001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022
023import java.util.ArrayList;
024import java.util.Collections;
025import java.util.List;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.HRegionLocation;
031import org.apache.hadoop.hbase.MetaRegionLocationCache;
032import org.apache.hadoop.hbase.MultithreadedTestUtil;
033import org.apache.hadoop.hbase.ServerName;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.master.HMaster;
036import org.apache.hadoop.hbase.master.RegionState;
037import org.apache.hadoop.hbase.testclassification.MasterTests;
038import org.apache.hadoop.hbase.testclassification.SmallTests;
039import org.apache.hadoop.hbase.util.JVMClusterUtil;
040import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
041import org.apache.hadoop.hbase.zookeeper.ZKUtil;
042import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
043import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
044import org.junit.AfterClass;
045import org.junit.BeforeClass;
046import org.junit.ClassRule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049
050import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
051
052@Category({ SmallTests.class, MasterTests.class })
053public class TestMetaRegionLocationCache {
054  @ClassRule
055  public static final HBaseClassTestRule CLASS_RULE =
056    HBaseClassTestRule.forClass(TestMetaRegionLocationCache.class);
057
058  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
059  private static ConnectionRegistry REGISTRY;
060
061  @BeforeClass
062  public static void setUp() throws Exception {
063    TEST_UTIL.startMiniCluster(3);
064    HBaseTestingUtility.setReplicas(TEST_UTIL.getAdmin(), TableName.META_TABLE_NAME, 3);
065    REGISTRY = ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration());
066    RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(TEST_UTIL, REGISTRY);
067    TEST_UTIL.getAdmin().balancerSwitch(false, true);
068  }
069
070  @AfterClass
071  public static void cleanUp() throws Exception {
072    Closeables.close(REGISTRY, true);
073    TEST_UTIL.shutdownMiniCluster();
074  }
075
076  private List<HRegionLocation> getCurrentMetaLocations(ZKWatcher zk) throws Exception {
077    List<HRegionLocation> result = new ArrayList<>();
078    for (String znode : zk.getMetaReplicaNodes()) {
079      String path = ZNodePaths.joinZNode(zk.getZNodePaths().baseZNode, znode);
080      int replicaId = zk.getZNodePaths().getMetaReplicaIdFromPath(path);
081      RegionState state = MetaTableLocator.getMetaRegionState(zk, replicaId);
082      result.add(new HRegionLocation(state.getRegion(), state.getServerName()));
083    }
084    return result;
085  }
086
087  // Verifies that the cached meta locations in the given master are in sync with what is in ZK.
088  private void verifyCachedMetaLocations(HMaster master) throws Exception {
089    // Wait until initial meta locations are loaded.
090    int retries = 0;
091    while (!master.getMetaRegionLocationCache().getMetaRegionLocations().isPresent()) {
092      Thread.sleep(1000);
093      if (++retries == 10) {
094        break;
095      }
096    }
097    ZKWatcher zk = master.getZooKeeper();
098    List<String> metaZnodes = zk.getMetaReplicaNodes();
099    // Wait till all replicas available.
100    retries = 0;
101    while (
102      master.getMetaRegionLocationCache().getMetaRegionLocations().get().size() != metaZnodes.size()
103    ) {
104      Thread.sleep(1000);
105      if (++retries == 10) {
106        break;
107      }
108    }
109    List<HRegionLocation> metaHRLs =
110      master.getMetaRegionLocationCache().getMetaRegionLocations().get();
111    assertFalse(metaHRLs.isEmpty());
112    assertEquals(metaZnodes.size(), metaHRLs.size());
113    List<HRegionLocation> actualHRLs = getCurrentMetaLocations(zk);
114    Collections.sort(metaHRLs);
115    Collections.sort(actualHRLs);
116    assertEquals(actualHRLs, metaHRLs);
117  }
118
119  @Test
120  public void testInitialMetaLocations() throws Exception {
121    verifyCachedMetaLocations(TEST_UTIL.getMiniHBaseCluster().getMaster());
122  }
123
124  @Test
125  public void testStandByMetaLocations() throws Exception {
126    HMaster standBy = TEST_UTIL.getMiniHBaseCluster().startMaster().getMaster();
127    verifyCachedMetaLocations(standBy);
128  }
129
130  /*
131   * Shuffles the meta region replicas around the cluster and makes sure the cache is not stale.
132   */
133  @Test
134  public void testMetaLocationsChange() throws Exception {
135    List<HRegionLocation> currentMetaLocs =
136      getCurrentMetaLocations(TEST_UTIL.getMiniHBaseCluster().getMaster().getZooKeeper());
137    // Move these replicas to random servers.
138    for (HRegionLocation location : currentMetaLocs) {
139      RegionReplicaTestHelper.moveRegion(TEST_UTIL, location);
140    }
141    RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(TEST_UTIL, REGISTRY);
142    for (JVMClusterUtil.MasterThread masterThread : TEST_UTIL.getMiniHBaseCluster()
143      .getMasterThreads()) {
144      verifyCachedMetaLocations(masterThread.getMaster());
145    }
146  }
147
148  /**
149   * Tests MetaRegionLocationCache's init procedure to make sure that it correctly watches the base
150   * znode for notifications.
151   */
152  @Test
153  public void testMetaRegionLocationCache() throws Exception {
154    final String parentZnodeName = "/randomznodename";
155    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
156    conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parentZnodeName);
157    ServerName sn = ServerName.valueOf("localhost", 1234, 5678);
158    try (ZKWatcher zkWatcher = new ZKWatcher(conf, null, null, true)) {
159      // A thread that repeatedly creates and drops an unrelated child znode. This is to simulate
160      // some ZK activity in the background.
161      MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
162      ctx.addThread(new MultithreadedTestUtil.RepeatingTestThread(ctx) {
163        @Override
164        public void doAnAction() throws Exception {
165          final String testZnode = parentZnodeName + "/child";
166          ZKUtil.createNodeIfNotExistsAndWatch(zkWatcher, testZnode, testZnode.getBytes());
167          ZKUtil.deleteNode(zkWatcher, testZnode);
168        }
169      });
170      ctx.startThreads();
171      try {
172        MetaRegionLocationCache metaCache = new MetaRegionLocationCache(zkWatcher);
173        // meta znodes do not exist at this point, cache should be empty.
174        assertFalse(metaCache.getMetaRegionLocations().isPresent());
175        // Set the meta locations for a random meta replicas, simulating an active hmaster meta
176        // assignment.
177        for (int i = 0; i < 3; i++) {
178          // Updates the meta znodes.
179          MetaTableLocator.setMetaLocation(zkWatcher, sn, i, RegionState.State.OPEN);
180        }
181        // Wait until the meta cache is populated.
182        int iters = 0;
183        while (iters++ < 10) {
184          if (
185            metaCache.getMetaRegionLocations().isPresent()
186              && metaCache.getMetaRegionLocations().get().size() == 3
187          ) {
188            break;
189          }
190          Thread.sleep(1000);
191        }
192        List<HRegionLocation> metaLocations = metaCache.getMetaRegionLocations().get();
193        assertEquals(3, metaLocations.size());
194        for (HRegionLocation location : metaLocations) {
195          assertEquals(sn, location.getServerName());
196        }
197      } finally {
198        // clean up.
199        ctx.stop();
200        ZKUtil.deleteChildrenRecursively(zkWatcher, parentZnodeName);
201      }
202    }
203  }
204}