001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.client; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertTrue; 023 024import java.util.ArrayList; 025import java.util.Collections; 026import java.util.List; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseTestingUtil; 030import org.apache.hadoop.hbase.HConstants; 031import org.apache.hadoop.hbase.HRegionLocation; 032import org.apache.hadoop.hbase.MetaRegionLocationCache; 033import org.apache.hadoop.hbase.MultithreadedTestUtil; 034import org.apache.hadoop.hbase.ServerName; 035import org.apache.hadoop.hbase.TableName; 036import org.apache.hadoop.hbase.master.HMaster; 037import org.apache.hadoop.hbase.master.RegionState; 038import org.apache.hadoop.hbase.security.User; 039import org.apache.hadoop.hbase.testclassification.MasterTests; 040import org.apache.hadoop.hbase.testclassification.SmallTests; 041import org.apache.hadoop.hbase.util.JVMClusterUtil; 042import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; 043import org.apache.hadoop.hbase.zookeeper.ZKUtil; 044import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 045import org.apache.hadoop.hbase.zookeeper.ZNodePaths; 046import org.junit.AfterClass; 047import org.junit.BeforeClass; 048import org.junit.ClassRule; 049import org.junit.Test; 050import org.junit.experimental.categories.Category; 051 052import org.apache.hbase.thirdparty.com.google.common.io.Closeables; 053 054@Category({ SmallTests.class, MasterTests.class }) 055public class TestMetaRegionLocationCache { 056 @ClassRule 057 public static final HBaseClassTestRule CLASS_RULE = 058 HBaseClassTestRule.forClass(TestMetaRegionLocationCache.class); 059 060 private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); 061 private static ConnectionRegistry REGISTRY; 062 063 @BeforeClass 064 public static void setUp() throws Exception { 065 TEST_UTIL.startMiniCluster(3); 066 HBaseTestingUtil.setReplicas(TEST_UTIL.getAdmin(), TableName.META_TABLE_NAME, 3); 067 REGISTRY = 068 ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration(), User.getCurrent()); 069 RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(TEST_UTIL, REGISTRY); 070 TEST_UTIL.getAdmin().balancerSwitch(false, true); 071 } 072 073 @AfterClass 074 public static void cleanUp() throws Exception { 075 Closeables.close(REGISTRY, true); 076 TEST_UTIL.shutdownMiniCluster(); 077 } 078 079 private List<HRegionLocation> getCurrentMetaLocations(ZKWatcher zk) throws Exception { 080 List<HRegionLocation> result = new ArrayList<>(); 081 for (String znode : zk.getMetaReplicaNodes()) { 082 String path = ZNodePaths.joinZNode(zk.getZNodePaths().baseZNode, znode); 083 int replicaId = zk.getZNodePaths().getMetaReplicaIdFromPath(path); 084 RegionState state = MetaTableLocator.getMetaRegionState(zk, replicaId); 085 result.add(new HRegionLocation(state.getRegion(), state.getServerName())); 086 } 087 return result; 088 } 089 090 // Verifies that the cached meta locations in the given master are in sync with what is in ZK. 091 private void verifyCachedMetaLocations(HMaster master) throws Exception { 092 // Wait until initial meta locations are loaded. 093 int retries = 0; 094 while (master.getMetaRegionLocationCache().getMetaRegionLocations().isEmpty()) { 095 Thread.sleep(1000); 096 if (++retries == 10) { 097 break; 098 } 099 } 100 ZKWatcher zk = master.getZooKeeper(); 101 List<String> metaZnodes = zk.getMetaReplicaNodes(); 102 // Wait till all replicas available. 103 retries = 0; 104 while ( 105 master.getMetaRegionLocationCache().getMetaRegionLocations().size() != metaZnodes.size() 106 ) { 107 Thread.sleep(1000); 108 if (++retries == 10) { 109 break; 110 } 111 } 112 List<HRegionLocation> metaHRLs = master.getMetaRegionLocationCache().getMetaRegionLocations(); 113 assertFalse(metaHRLs.isEmpty()); 114 assertEquals(metaZnodes.size(), metaHRLs.size()); 115 List<HRegionLocation> actualHRLs = getCurrentMetaLocations(zk); 116 Collections.sort(metaHRLs); 117 Collections.sort(actualHRLs); 118 assertEquals(actualHRLs, metaHRLs); 119 } 120 121 @Test 122 public void testInitialMetaLocations() throws Exception { 123 verifyCachedMetaLocations(TEST_UTIL.getMiniHBaseCluster().getMaster()); 124 } 125 126 @Test 127 public void testStandByMetaLocations() throws Exception { 128 HMaster standBy = TEST_UTIL.getMiniHBaseCluster().startMaster().getMaster(); 129 verifyCachedMetaLocations(standBy); 130 } 131 132 /* 133 * Shuffles the meta region replicas around the cluster and makes sure the cache is not stale. 134 */ 135 @Test 136 public void testMetaLocationsChange() throws Exception { 137 List<HRegionLocation> currentMetaLocs = 138 getCurrentMetaLocations(TEST_UTIL.getMiniHBaseCluster().getMaster().getZooKeeper()); 139 // Move these replicas to random servers. 140 for (HRegionLocation location : currentMetaLocs) { 141 RegionReplicaTestHelper.moveRegion(TEST_UTIL, location); 142 } 143 RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(TEST_UTIL, REGISTRY); 144 for (JVMClusterUtil.MasterThread masterThread : TEST_UTIL.getMiniHBaseCluster() 145 .getMasterThreads()) { 146 verifyCachedMetaLocations(masterThread.getMaster()); 147 } 148 } 149 150 /** 151 * Tests MetaRegionLocationCache's init procedure to make sure that it correctly watches the base 152 * znode for notifications. 153 */ 154 @Test 155 public void testMetaRegionLocationCache() throws Exception { 156 final String parentZnodeName = "/randomznodename"; 157 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); 158 conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parentZnodeName); 159 ServerName sn = ServerName.valueOf("localhost", 1234, 5678); 160 try (ZKWatcher zkWatcher = new ZKWatcher(conf, null, null, true)) { 161 // A thread that repeatedly creates and drops an unrelated child znode. This is to simulate 162 // some ZK activity in the background. 163 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf); 164 ctx.addThread(new MultithreadedTestUtil.RepeatingTestThread(ctx) { 165 @Override 166 public void doAnAction() throws Exception { 167 final String testZnode = parentZnodeName + "/child"; 168 ZKUtil.createNodeIfNotExistsAndWatch(zkWatcher, testZnode, testZnode.getBytes()); 169 ZKUtil.deleteNode(zkWatcher, testZnode); 170 } 171 }); 172 ctx.startThreads(); 173 try { 174 MetaRegionLocationCache metaCache = new MetaRegionLocationCache(zkWatcher); 175 // meta znodes do not exist at this point, cache should be empty. 176 assertTrue(metaCache.getMetaRegionLocations().isEmpty()); 177 // Set the meta locations for a random meta replicas, simulating an active hmaster meta 178 // assignment. 179 for (int i = 0; i < 3; i++) { 180 // Updates the meta znodes. 181 MetaTableLocator.setMetaLocation(zkWatcher, sn, i, RegionState.State.OPEN); 182 } 183 // Wait until the meta cache is populated. 184 int iters = 0; 185 while (iters++ < 10) { 186 if (metaCache.getMetaRegionLocations().size() == 3) { 187 break; 188 } 189 Thread.sleep(1000); 190 } 191 List<HRegionLocation> metaLocations = metaCache.getMetaRegionLocations(); 192 assertEquals(3, metaLocations.size()); 193 for (HRegionLocation location : metaLocations) { 194 assertEquals(sn, location.getServerName()); 195 } 196 } finally { 197 // clean up. 198 ctx.stop(); 199 ZKUtil.deleteChildrenRecursively(zkWatcher, parentZnodeName); 200 } 201 } 202 } 203}