001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import java.util.ArrayList;
023import java.util.Collections;
024import java.util.List;
025import org.apache.commons.io.IOUtils;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.HRegionLocation;
031import org.apache.hadoop.hbase.MultithreadedTestUtil;
032import org.apache.hadoop.hbase.ServerName;
033import org.apache.hadoop.hbase.master.HMaster;
034import org.apache.hadoop.hbase.master.MetaRegionLocationCache;
035import org.apache.hadoop.hbase.master.RegionState;
036import org.apache.hadoop.hbase.testclassification.MasterTests;
037import org.apache.hadoop.hbase.testclassification.SmallTests;
038import org.apache.hadoop.hbase.util.JVMClusterUtil;
039import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
040import org.apache.hadoop.hbase.zookeeper.ZKUtil;
041import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
042import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
043import org.junit.AfterClass;
044import org.junit.BeforeClass;
045import org.junit.ClassRule;
046import org.junit.Test;
047import org.junit.experimental.categories.Category;
048
049@Category({SmallTests.class, MasterTests.class })
050public class TestMetaRegionLocationCache {
051  @ClassRule
052  public static final HBaseClassTestRule CLASS_RULE =
053      HBaseClassTestRule.forClass(TestMetaRegionLocationCache.class);
054
055  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
056  private static ConnectionRegistry REGISTRY;
057
058  @BeforeClass
059  public static void setUp() throws Exception {
060    TEST_UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM, 3);
061    TEST_UTIL.startMiniCluster(3);
062    REGISTRY = ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration());
063    RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(TEST_UTIL, REGISTRY);
064    TEST_UTIL.getAdmin().balancerSwitch(false, true);
065  }
066
067  @AfterClass
068  public static void cleanUp() throws Exception {
069    IOUtils.closeQuietly(REGISTRY);
070    TEST_UTIL.shutdownMiniCluster();
071  }
072
073  private List<HRegionLocation> getCurrentMetaLocations(ZKWatcher zk) throws Exception {
074    List<HRegionLocation> result = new ArrayList<>();
075    for (String znode: zk.getMetaReplicaNodes()) {
076      String path = ZNodePaths.joinZNode(zk.getZNodePaths().baseZNode, znode);
077      int replicaId = zk.getZNodePaths().getMetaReplicaIdFromPath(path);
078      RegionState state = MetaTableLocator.getMetaRegionState(zk, replicaId);
079      result.add(new HRegionLocation(state.getRegion(), state.getServerName()));
080    }
081    return result;
082  }
083
084  // Verifies that the cached meta locations in the given master are in sync with what is in ZK.
085  private void verifyCachedMetaLocations(HMaster master) throws Exception {
086    // Wait until initial meta locations are loaded.
087    int retries = 0;
088    while (!master.getMetaRegionLocationCache().getMetaRegionLocations().isPresent()) {
089      Thread.sleep(1000);
090      if (++retries == 10) {
091        break;
092      }
093    }
094    List<HRegionLocation> metaHRLs =
095        master.getMetaRegionLocationCache().getMetaRegionLocations().get();
096    assertFalse(metaHRLs.isEmpty());
097    ZKWatcher zk = master.getZooKeeper();
098    List<String> metaZnodes = zk.getMetaReplicaNodes();
099    assertEquals(metaZnodes.size(), metaHRLs.size());
100    List<HRegionLocation> actualHRLs = getCurrentMetaLocations(zk);
101    Collections.sort(metaHRLs);
102    Collections.sort(actualHRLs);
103    assertEquals(actualHRLs, metaHRLs);
104  }
105
106  @Test public void testInitialMetaLocations() throws Exception {
107    verifyCachedMetaLocations(TEST_UTIL.getMiniHBaseCluster().getMaster());
108  }
109
110  @Test public void testStandByMetaLocations() throws Exception {
111    HMaster standBy = TEST_UTIL.getMiniHBaseCluster().startMaster().getMaster();
112    verifyCachedMetaLocations(standBy);
113  }
114
115  /*
116   * Shuffles the meta region replicas around the cluster and makes sure the cache is not stale.
117   */
118  @Test public void testMetaLocationsChange() throws Exception {
119    List<HRegionLocation> currentMetaLocs =
120        getCurrentMetaLocations(TEST_UTIL.getMiniHBaseCluster().getMaster().getZooKeeper());
121    // Move these replicas to random servers.
122    for (HRegionLocation location: currentMetaLocs) {
123      RegionReplicaTestHelper.moveRegion(TEST_UTIL, location);
124    }
125    RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(TEST_UTIL, REGISTRY);
126    for (JVMClusterUtil.MasterThread masterThread:
127        TEST_UTIL.getMiniHBaseCluster().getMasterThreads()) {
128      verifyCachedMetaLocations(masterThread.getMaster());
129    }
130  }
131
132  /**
133   * Tests MetaRegionLocationCache's init procedure to make sure that it correctly watches the base
134   * znode for notifications.
135   */
136  @Test public void testMetaRegionLocationCache() throws Exception {
137    final String parentZnodeName = "/randomznodename";
138    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
139    conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parentZnodeName);
140    ServerName sn = ServerName.valueOf("localhost", 1234, 5678);
141    try (ZKWatcher zkWatcher = new ZKWatcher(conf, null, null, true)) {
142      // A thread that repeatedly creates and drops an unrelated child znode. This is to simulate
143      // some ZK activity in the background.
144      MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
145      ctx.addThread(new MultithreadedTestUtil.RepeatingTestThread(ctx) {
146        @Override public void doAnAction() throws Exception {
147          final String testZnode = parentZnodeName + "/child";
148          ZKUtil.createNodeIfNotExistsAndWatch(zkWatcher, testZnode, testZnode.getBytes());
149          ZKUtil.deleteNode(zkWatcher, testZnode);
150        }
151      });
152      ctx.startThreads();
153      try {
154        MetaRegionLocationCache metaCache = new MetaRegionLocationCache(zkWatcher);
155        // meta znodes do not exist at this point, cache should be empty.
156        assertFalse(metaCache.getMetaRegionLocations().isPresent());
157        // Set the meta locations for a random meta replicas, simulating an active hmaster meta
158        // assignment.
159        for (int i = 0; i < 3; i++) {
160          // Updates the meta znodes.
161          MetaTableLocator.setMetaLocation(zkWatcher, sn, i, RegionState.State.OPEN);
162        }
163        // Wait until the meta cache is populated.
164        int iters = 0;
165        while (iters++ < 10) {
166          if (metaCache.getMetaRegionLocations().isPresent()
167            && metaCache.getMetaRegionLocations().get().size() == 3) {
168            break;
169          }
170          Thread.sleep(1000);
171        }
172        List<HRegionLocation> metaLocations = metaCache.getMetaRegionLocations().get();
173        assertEquals(3, metaLocations.size());
174        for (HRegionLocation location : metaLocations) {
175          assertEquals(sn, location.getServerName());
176        }
177      } finally {
178        // clean up.
179        ctx.stop();
180        ZKUtil.deleteChildrenRecursively(zkWatcher, parentZnodeName);
181      }
182    }
183  }
184}