001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.util.ArrayList;
025import java.util.Collections;
026import java.util.List;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.HRegionLocation;
032import org.apache.hadoop.hbase.MetaRegionLocationCache;
033import org.apache.hadoop.hbase.MultithreadedTestUtil;
034import org.apache.hadoop.hbase.ServerName;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.master.HMaster;
037import org.apache.hadoop.hbase.master.RegionState;
038import org.apache.hadoop.hbase.security.User;
039import org.apache.hadoop.hbase.testclassification.MasterTests;
040import org.apache.hadoop.hbase.testclassification.SmallTests;
041import org.apache.hadoop.hbase.util.JVMClusterUtil;
042import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
043import org.apache.hadoop.hbase.zookeeper.ZKUtil;
044import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
045import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
046import org.junit.AfterClass;
047import org.junit.BeforeClass;
048import org.junit.ClassRule;
049import org.junit.Test;
050import org.junit.experimental.categories.Category;
051
052import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
053
054@Category({ SmallTests.class, MasterTests.class })
055public class TestMetaRegionLocationCache {
056  @ClassRule
057  public static final HBaseClassTestRule CLASS_RULE =
058    HBaseClassTestRule.forClass(TestMetaRegionLocationCache.class);
059
060  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
061  private static ConnectionRegistry REGISTRY;
062
063  @BeforeClass
064  public static void setUp() throws Exception {
065    TEST_UTIL.startMiniCluster(3);
066    HBaseTestingUtil.setReplicas(TEST_UTIL.getAdmin(), TableName.META_TABLE_NAME, 3);
067    REGISTRY =
068      ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration(), User.getCurrent());
069    RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(TEST_UTIL, REGISTRY);
070    TEST_UTIL.getAdmin().balancerSwitch(false, true);
071  }
072
073  @AfterClass
074  public static void cleanUp() throws Exception {
075    Closeables.close(REGISTRY, true);
076    TEST_UTIL.shutdownMiniCluster();
077  }
078
079  private List<HRegionLocation> getCurrentMetaLocations(ZKWatcher zk) throws Exception {
080    List<HRegionLocation> result = new ArrayList<>();
081    for (String znode : zk.getMetaReplicaNodes()) {
082      String path = ZNodePaths.joinZNode(zk.getZNodePaths().baseZNode, znode);
083      int replicaId = zk.getZNodePaths().getMetaReplicaIdFromPath(path);
084      RegionState state = MetaTableLocator.getMetaRegionState(zk, replicaId);
085      result.add(new HRegionLocation(state.getRegion(), state.getServerName()));
086    }
087    return result;
088  }
089
090  // Verifies that the cached meta locations in the given master are in sync with what is in ZK.
091  private void verifyCachedMetaLocations(HMaster master) throws Exception {
092    // Wait until initial meta locations are loaded.
093    int retries = 0;
094    while (master.getMetaRegionLocationCache().getMetaRegionLocations().isEmpty()) {
095      Thread.sleep(1000);
096      if (++retries == 10) {
097        break;
098      }
099    }
100    ZKWatcher zk = master.getZooKeeper();
101    List<String> metaZnodes = zk.getMetaReplicaNodes();
102    // Wait till all replicas available.
103    retries = 0;
104    while (
105      master.getMetaRegionLocationCache().getMetaRegionLocations().size() != metaZnodes.size()
106    ) {
107      Thread.sleep(1000);
108      if (++retries == 10) {
109        break;
110      }
111    }
112    List<HRegionLocation> metaHRLs = master.getMetaRegionLocationCache().getMetaRegionLocations();
113    assertFalse(metaHRLs.isEmpty());
114    assertEquals(metaZnodes.size(), metaHRLs.size());
115    List<HRegionLocation> actualHRLs = getCurrentMetaLocations(zk);
116    Collections.sort(metaHRLs);
117    Collections.sort(actualHRLs);
118    assertEquals(actualHRLs, metaHRLs);
119  }
120
121  @Test
122  public void testInitialMetaLocations() throws Exception {
123    verifyCachedMetaLocations(TEST_UTIL.getMiniHBaseCluster().getMaster());
124  }
125
126  @Test
127  public void testStandByMetaLocations() throws Exception {
128    HMaster standBy = TEST_UTIL.getMiniHBaseCluster().startMaster().getMaster();
129    verifyCachedMetaLocations(standBy);
130  }
131
132  /*
133   * Shuffles the meta region replicas around the cluster and makes sure the cache is not stale.
134   */
135  @Test
136  public void testMetaLocationsChange() throws Exception {
137    List<HRegionLocation> currentMetaLocs =
138      getCurrentMetaLocations(TEST_UTIL.getMiniHBaseCluster().getMaster().getZooKeeper());
139    // Move these replicas to random servers.
140    for (HRegionLocation location : currentMetaLocs) {
141      RegionReplicaTestHelper.moveRegion(TEST_UTIL, location);
142    }
143    RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(TEST_UTIL, REGISTRY);
144    for (JVMClusterUtil.MasterThread masterThread : TEST_UTIL.getMiniHBaseCluster()
145      .getMasterThreads()) {
146      verifyCachedMetaLocations(masterThread.getMaster());
147    }
148  }
149
150  /**
151   * Tests MetaRegionLocationCache's init procedure to make sure that it correctly watches the base
152   * znode for notifications.
153   */
154  @Test
155  public void testMetaRegionLocationCache() throws Exception {
156    final String parentZnodeName = "/randomznodename";
157    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
158    conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parentZnodeName);
159    ServerName sn = ServerName.valueOf("localhost", 1234, 5678);
160    try (ZKWatcher zkWatcher = new ZKWatcher(conf, null, null, true)) {
161      // A thread that repeatedly creates and drops an unrelated child znode. This is to simulate
162      // some ZK activity in the background.
163      MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
164      ctx.addThread(new MultithreadedTestUtil.RepeatingTestThread(ctx) {
165        @Override
166        public void doAnAction() throws Exception {
167          final String testZnode = parentZnodeName + "/child";
168          ZKUtil.createNodeIfNotExistsAndWatch(zkWatcher, testZnode, testZnode.getBytes());
169          ZKUtil.deleteNode(zkWatcher, testZnode);
170        }
171      });
172      ctx.startThreads();
173      try {
174        MetaRegionLocationCache metaCache = new MetaRegionLocationCache(zkWatcher);
175        // meta znodes do not exist at this point, cache should be empty.
176        assertTrue(metaCache.getMetaRegionLocations().isEmpty());
177        // Set the meta locations for a random meta replicas, simulating an active hmaster meta
178        // assignment.
179        for (int i = 0; i < 3; i++) {
180          // Updates the meta znodes.
181          MetaTableLocator.setMetaLocation(zkWatcher, sn, i, RegionState.State.OPEN);
182        }
183        // Wait until the meta cache is populated.
184        int iters = 0;
185        while (iters++ < 10) {
186          if (metaCache.getMetaRegionLocations().size() == 3) {
187            break;
188          }
189          Thread.sleep(1000);
190        }
191        List<HRegionLocation> metaLocations = metaCache.getMetaRegionLocations();
192        assertEquals(3, metaLocations.size());
193        for (HRegionLocation location : metaLocations) {
194          assertEquals(sn, location.getServerName());
195        }
196      } finally {
197        // clean up.
198        ctx.stop();
199        ZKUtil.deleteChildrenRecursively(zkWatcher, parentZnodeName);
200      }
201    }
202  }
203}