001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.client; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022 023import java.util.ArrayList; 024import java.util.Collections; 025import java.util.List; 026import org.apache.commons.io.IOUtils; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseTestingUtility; 030import org.apache.hadoop.hbase.HConstants; 031import org.apache.hadoop.hbase.HRegionLocation; 032import org.apache.hadoop.hbase.MultithreadedTestUtil; 033import org.apache.hadoop.hbase.ServerName; 034import org.apache.hadoop.hbase.TableName; 035import org.apache.hadoop.hbase.master.HMaster; 036import org.apache.hadoop.hbase.master.MetaRegionLocationCache; 037import org.apache.hadoop.hbase.master.RegionState; 038import org.apache.hadoop.hbase.testclassification.MasterTests; 039import org.apache.hadoop.hbase.testclassification.SmallTests; 040import org.apache.hadoop.hbase.util.JVMClusterUtil; 041import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; 042import org.apache.hadoop.hbase.zookeeper.ZKUtil; 043import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 044import org.apache.hadoop.hbase.zookeeper.ZNodePaths; 045import org.junit.AfterClass; 046import org.junit.BeforeClass; 047import org.junit.ClassRule; 048import org.junit.Test; 049import org.junit.experimental.categories.Category; 050 051@Category({SmallTests.class, MasterTests.class }) 052public class TestMetaRegionLocationCache { 053 @ClassRule 054 public static final HBaseClassTestRule CLASS_RULE = 055 HBaseClassTestRule.forClass(TestMetaRegionLocationCache.class); 056 057 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 058 private static ConnectionRegistry REGISTRY; 059 060 @BeforeClass 061 public static void setUp() throws Exception { 062 TEST_UTIL.startMiniCluster(3); 063 HBaseTestingUtility.setReplicas(TEST_UTIL.getAdmin(), TableName.META_TABLE_NAME, 3); 064 REGISTRY = ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration()); 065 RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(TEST_UTIL, REGISTRY); 066 TEST_UTIL.getAdmin().balancerSwitch(false, true); 067 } 068 069 @AfterClass 070 public static void cleanUp() throws Exception { 071 IOUtils.closeQuietly(REGISTRY); 072 TEST_UTIL.shutdownMiniCluster(); 073 } 074 075 private List<HRegionLocation> getCurrentMetaLocations(ZKWatcher zk) throws Exception { 076 List<HRegionLocation> result = new ArrayList<>(); 077 for (String znode: zk.getMetaReplicaNodes()) { 078 String path = ZNodePaths.joinZNode(zk.getZNodePaths().baseZNode, znode); 079 int replicaId = zk.getZNodePaths().getMetaReplicaIdFromPath(path); 080 RegionState state = MetaTableLocator.getMetaRegionState(zk, replicaId); 081 result.add(new HRegionLocation(state.getRegion(), state.getServerName())); 082 } 083 return result; 084 } 085 086 // Verifies that the cached meta locations in the given master are in sync with what is in ZK. 087 private void verifyCachedMetaLocations(HMaster master) throws Exception { 088 // Wait until initial meta locations are loaded. 089 int retries = 0; 090 while (!master.getMetaRegionLocationCache().getMetaRegionLocations().isPresent()) { 091 Thread.sleep(1000); 092 if (++retries == 10) { 093 break; 094 } 095 } 096 List<HRegionLocation> metaHRLs = 097 master.getMetaRegionLocationCache().getMetaRegionLocations().get(); 098 assertFalse(metaHRLs.isEmpty()); 099 ZKWatcher zk = master.getZooKeeper(); 100 List<String> metaZnodes = zk.getMetaReplicaNodes(); 101 assertEquals(metaZnodes.size(), metaHRLs.size()); 102 List<HRegionLocation> actualHRLs = getCurrentMetaLocations(zk); 103 Collections.sort(metaHRLs); 104 Collections.sort(actualHRLs); 105 assertEquals(actualHRLs, metaHRLs); 106 } 107 108 @Test public void testInitialMetaLocations() throws Exception { 109 verifyCachedMetaLocations(TEST_UTIL.getMiniHBaseCluster().getMaster()); 110 } 111 112 @Test public void testStandByMetaLocations() throws Exception { 113 HMaster standBy = TEST_UTIL.getMiniHBaseCluster().startMaster().getMaster(); 114 verifyCachedMetaLocations(standBy); 115 } 116 117 /* 118 * Shuffles the meta region replicas around the cluster and makes sure the cache is not stale. 119 */ 120 @Test public void testMetaLocationsChange() throws Exception { 121 List<HRegionLocation> currentMetaLocs = 122 getCurrentMetaLocations(TEST_UTIL.getMiniHBaseCluster().getMaster().getZooKeeper()); 123 // Move these replicas to random servers. 124 for (HRegionLocation location: currentMetaLocs) { 125 RegionReplicaTestHelper.moveRegion(TEST_UTIL, location); 126 } 127 RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(TEST_UTIL, REGISTRY); 128 for (JVMClusterUtil.MasterThread masterThread: 129 TEST_UTIL.getMiniHBaseCluster().getMasterThreads()) { 130 verifyCachedMetaLocations(masterThread.getMaster()); 131 } 132 } 133 134 /** 135 * Tests MetaRegionLocationCache's init procedure to make sure that it correctly watches the base 136 * znode for notifications. 137 */ 138 @Test public void testMetaRegionLocationCache() throws Exception { 139 final String parentZnodeName = "/randomznodename"; 140 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); 141 conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parentZnodeName); 142 ServerName sn = ServerName.valueOf("localhost", 1234, 5678); 143 try (ZKWatcher zkWatcher = new ZKWatcher(conf, null, null, true)) { 144 // A thread that repeatedly creates and drops an unrelated child znode. This is to simulate 145 // some ZK activity in the background. 146 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf); 147 ctx.addThread(new MultithreadedTestUtil.RepeatingTestThread(ctx) { 148 @Override public void doAnAction() throws Exception { 149 final String testZnode = parentZnodeName + "/child"; 150 ZKUtil.createNodeIfNotExistsAndWatch(zkWatcher, testZnode, testZnode.getBytes()); 151 ZKUtil.deleteNode(zkWatcher, testZnode); 152 } 153 }); 154 ctx.startThreads(); 155 try { 156 MetaRegionLocationCache metaCache = new MetaRegionLocationCache(zkWatcher); 157 // meta znodes do not exist at this point, cache should be empty. 158 assertFalse(metaCache.getMetaRegionLocations().isPresent()); 159 // Set the meta locations for a random meta replicas, simulating an active hmaster meta 160 // assignment. 161 for (int i = 0; i < 3; i++) { 162 // Updates the meta znodes. 163 MetaTableLocator.setMetaLocation(zkWatcher, sn, i, RegionState.State.OPEN); 164 } 165 // Wait until the meta cache is populated. 166 int iters = 0; 167 while (iters++ < 10) { 168 if (metaCache.getMetaRegionLocations().isPresent() 169 && metaCache.getMetaRegionLocations().get().size() == 3) { 170 break; 171 } 172 Thread.sleep(1000); 173 } 174 List<HRegionLocation> metaLocations = metaCache.getMetaRegionLocations().get(); 175 assertEquals(3, metaLocations.size()); 176 for (HRegionLocation location : metaLocations) { 177 assertEquals(sn, location.getServerName()); 178 } 179 } finally { 180 // clean up. 181 ctx.stop(); 182 ZKUtil.deleteChildrenRecursively(zkWatcher, parentZnodeName); 183 } 184 } 185 } 186}