001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.client; 019 020import static org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position.PRIMARY; 021import static org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position.SECONDARY; 022import static org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position.TERTIARY; 023import static org.junit.Assert.assertArrayEquals; 024import static org.junit.Assert.assertEquals; 025import static org.junit.Assert.assertNotNull; 026import static org.junit.Assert.assertNull; 027import static org.junit.Assert.assertTrue; 028 029import java.io.IOException; 030import java.net.InetSocketAddress; 031import java.util.List; 032import java.util.Map; 033import java.util.concurrent.TimeUnit; 034import org.apache.hadoop.conf.Configuration; 035import org.apache.hadoop.hbase.HBaseClassTestRule; 036import org.apache.hadoop.hbase.HBaseTestingUtility; 037import org.apache.hadoop.hbase.HConstants; 038import org.apache.hadoop.hbase.HRegionInfo; 039import org.apache.hadoop.hbase.HRegionLocation; 040import org.apache.hadoop.hbase.MetaTableAccessor; 041import org.apache.hadoop.hbase.NamespaceDescriptor; 042import org.apache.hadoop.hbase.ServerName; 043import org.apache.hadoop.hbase.TableName; 044import org.apache.hadoop.hbase.Waiter; 045import org.apache.hadoop.hbase.favored.FavoredNodeAssignmentHelper; 046import org.apache.hadoop.hbase.favored.FavoredNodesManager; 047import org.apache.hadoop.hbase.master.LoadBalancer; 048import org.apache.hadoop.hbase.master.ServerManager; 049import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; 050import org.apache.hadoop.hbase.master.balancer.LoadOnlyFavoredStochasticBalancer; 051import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; 052import org.apache.hadoop.hbase.regionserver.HRegionServer; 053import org.apache.hadoop.hbase.testclassification.ClientTests; 054import org.apache.hadoop.hbase.testclassification.MediumTests; 055import org.apache.hadoop.hbase.util.Bytes; 056import org.apache.hadoop.hbase.util.JVMClusterUtil; 057import org.apache.hadoop.hbase.util.Threads; 058import org.junit.AfterClass; 059import org.junit.Before; 060import org.junit.BeforeClass; 061import org.junit.ClassRule; 062import org.junit.Rule; 063import org.junit.Test; 064import org.junit.experimental.categories.Category; 065import org.junit.rules.TestName; 066import org.slf4j.Logger; 067import org.slf4j.LoggerFactory; 068 069import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 070import org.apache.hbase.thirdparty.com.google.common.collect.Maps; 071 072@Category({ClientTests.class, MediumTests.class}) 073public class TestTableFavoredNodes { 074 075 @ClassRule 076 public static final HBaseClassTestRule CLASS_RULE = 077 HBaseClassTestRule.forClass(TestTableFavoredNodes.class); 078 079 private static final Logger LOG = LoggerFactory.getLogger(TestTableFavoredNodes.class); 080 081 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 082 private final static int WAIT_TIMEOUT = 60000; 083 private final static int SLAVES = 8; 084 private FavoredNodesManager fnm; 085 private Admin admin; 086 087 private final byte[][] splitKeys = new byte[][] {Bytes.toBytes(1), Bytes.toBytes(9)}; 088 private final int NUM_REGIONS = splitKeys.length + 1; 089 090 @Rule 091 public TestName name = new TestName(); 092 093 @BeforeClass 094 public static void setupBeforeClass() throws Exception { 095 Configuration conf = TEST_UTIL.getConfiguration(); 096 // Setting FavoredNodeBalancer will enable favored nodes 097 conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, 098 LoadOnlyFavoredStochasticBalancer.class, LoadBalancer.class); 099 conf.set(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, "" + SLAVES); 100 101 // This helps test if RS get the appropriate FN updates. 102 conf.set(BaseLoadBalancer.TABLES_ON_MASTER, "none"); 103 TEST_UTIL.startMiniCluster(SLAVES); 104 TEST_UTIL.getMiniHBaseCluster().waitForActiveAndReadyMaster(WAIT_TIMEOUT); 105 } 106 107 @AfterClass 108 public static void tearDownAfterClass() throws Exception { 109 TEST_UTIL.shutdownMiniCluster(); 110 TEST_UTIL.cleanupTestDir(); 111 } 112 113 @Before 114 public void setup() throws IOException { 115 fnm = TEST_UTIL.getMiniHBaseCluster().getMaster().getFavoredNodesManager(); 116 admin = TEST_UTIL.getAdmin(); 117 admin.setBalancerRunning(false, true); 118 admin.enableCatalogJanitor(false); 119 } 120 121 /* 122 * Create a table with FN enabled and check if all its regions have favored nodes set. 123 */ 124 @Test 125 public void testCreateTable() throws Exception { 126 final TableName tableName = TableName.valueOf(name.getMethodName()); 127 TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys); 128 TEST_UTIL.waitUntilAllRegionsAssigned(tableName); 129 130 // All regions should have favored nodes 131 checkIfFavoredNodeInformationIsCorrect(tableName); 132 133 List<HRegionInfo> regions = admin.getTableRegions(tableName); 134 135 TEST_UTIL.deleteTable(tableName); 136 137 checkNoFNForDeletedTable(regions); 138 } 139 140 /* 141 * Checks if favored node information is removed on table truncation. 142 */ 143 @Test 144 public void testTruncateTable() throws Exception { 145 final TableName tableName = TableName.valueOf(name.getMethodName()); 146 TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys); 147 TEST_UTIL.waitUntilAllRegionsAssigned(tableName); 148 149 // All regions should have favored nodes 150 checkIfFavoredNodeInformationIsCorrect(tableName); 151 152 List<HRegionInfo> regions = admin.getTableRegions(tableName); 153 TEST_UTIL.truncateTable(tableName, true); 154 155 checkNoFNForDeletedTable(regions); 156 checkIfFavoredNodeInformationIsCorrect(tableName); 157 158 regions = admin.getTableRegions(tableName); 159 TEST_UTIL.truncateTable(tableName, false); 160 checkNoFNForDeletedTable(regions); 161 162 TEST_UTIL.deleteTable(tableName); 163 } 164 165 /* 166 * Check if daughters inherit at-least 2 FN from parent after region split. 167 */ 168 @Test 169 public void testSplitTable() throws Exception { 170 final TableName tableName = TableName.valueOf(name.getMethodName()); 171 Table t = TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys); 172 TEST_UTIL.waitUntilAllRegionsAssigned(tableName); 173 final int numberOfRegions = admin.getTableRegions(t.getName()).size(); 174 175 checkIfFavoredNodeInformationIsCorrect(tableName); 176 177 byte[] splitPoint = Bytes.toBytes(0); 178 RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName); 179 HRegionInfo parent = locator.getRegionLocation(splitPoint).getRegionInfo(); 180 List<ServerName> parentFN = fnm.getFavoredNodes(parent); 181 assertNotNull("FN should not be null for region: " + parent, parentFN); 182 183 LOG.info("SPLITTING TABLE"); 184 admin.split(tableName, splitPoint); 185 186 TEST_UTIL.waitUntilNoRegionsInTransition(WAIT_TIMEOUT); 187 LOG.info("FINISHED WAITING ON RIT"); 188 waitUntilTableRegionCountReached(tableName, numberOfRegions + 1); 189 190 // All regions should have favored nodes checkIfFavoredNodeInformationIsCorrect(tableName); 191 192 // Get the daughters of parent. 193 HRegionInfo daughter1 = locator.getRegionLocation(parent.getStartKey(), true).getRegionInfo(); 194 List<ServerName> daughter1FN = fnm.getFavoredNodes(daughter1); 195 196 HRegionInfo daughter2 = locator.getRegionLocation(splitPoint, true).getRegionInfo(); 197 List<ServerName> daughter2FN = fnm.getFavoredNodes(daughter2); 198 199 checkIfDaughterInherits2FN(parentFN, daughter1FN); 200 checkIfDaughterInherits2FN(parentFN, daughter2FN); 201 202 assertEquals("Daughter's PRIMARY FN should be PRIMARY of parent", 203 parentFN.get(PRIMARY.ordinal()), daughter1FN.get(PRIMARY.ordinal())); 204 assertEquals("Daughter's SECONDARY FN should be SECONDARY of parent", 205 parentFN.get(SECONDARY.ordinal()), daughter1FN.get(SECONDARY.ordinal())); 206 207 assertEquals("Daughter's PRIMARY FN should be PRIMARY of parent", 208 parentFN.get(PRIMARY.ordinal()), daughter2FN.get(PRIMARY.ordinal())); 209 assertEquals("Daughter's SECONDARY FN should be TERTIARY of parent", 210 parentFN.get(TERTIARY.ordinal()), daughter2FN.get(SECONDARY.ordinal())); 211 212 // Major compact table and run catalog janitor. Parent's FN should be removed 213 TEST_UTIL.getMiniHBaseCluster().compact(tableName, true); 214 admin.runCatalogScan(); 215 // Catalog cleanup is async. Wait on procedure to finish up. 216 ProcedureTestingUtility.waitAllProcedures( 217 TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor()); 218 // assertEquals("Parent region should have been cleaned", 1, admin.runCatalogScan()); 219 assertNull("Parent FN should be null", fnm.getFavoredNodes(parent)); 220 221 List<HRegionInfo> regions = admin.getTableRegions(tableName); 222 // Split and Table Disable interfere with each other around region replicas 223 // TODO. Meantime pause a few seconds. 224 Threads.sleep(2000); 225 LOG.info("STARTING DELETE"); 226 TEST_UTIL.deleteTable(tableName); 227 228 checkNoFNForDeletedTable(regions); 229 } 230 231 /* 232 * Check if merged region inherits FN from one of its regions. 233 */ 234 @Test 235 public void testMergeTable() throws Exception { 236 final TableName tableName = TableName.valueOf(name.getMethodName()); 237 TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys); 238 TEST_UTIL.waitUntilAllRegionsAssigned(tableName); 239 240 checkIfFavoredNodeInformationIsCorrect(tableName); 241 242 RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName); 243 HRegionInfo regionA = locator.getRegionLocation(HConstants.EMPTY_START_ROW).getRegionInfo(); 244 HRegionInfo regionB = locator.getRegionLocation(splitKeys[0]).getRegionInfo(); 245 246 List<ServerName> regionAFN = fnm.getFavoredNodes(regionA); 247 LOG.info("regionA: " + regionA.getEncodedName() + " with FN: " + fnm.getFavoredNodes(regionA)); 248 LOG.info("regionB: " + regionA.getEncodedName() + " with FN: " + fnm.getFavoredNodes(regionB)); 249 250 int countOfRegions = MetaTableAccessor.getRegionCount(TEST_UTIL.getConfiguration(), tableName); 251 admin.mergeRegionsAsync(regionA.getEncodedNameAsBytes(), 252 regionB.getEncodedNameAsBytes(), false).get(60, TimeUnit.SECONDS); 253 254 TEST_UTIL.waitUntilNoRegionsInTransition(WAIT_TIMEOUT); 255 waitUntilTableRegionCountReached(tableName, countOfRegions - 1); 256 257 // All regions should have favored nodes 258 checkIfFavoredNodeInformationIsCorrect(tableName); 259 260 HRegionInfo mergedRegion = 261 locator.getRegionLocation(HConstants.EMPTY_START_ROW).getRegionInfo(); 262 List<ServerName> mergedFN = fnm.getFavoredNodes(mergedRegion); 263 264 assertArrayEquals("Merged region doesn't match regionA's FN", 265 regionAFN.toArray(), mergedFN.toArray()); 266 267 // Major compact table and run catalog janitor. Parent FN should be removed 268 TEST_UTIL.getMiniHBaseCluster().compact(tableName, true); 269 assertEquals("Merge parents should have been cleaned", 1, admin.runCatalogScan()); 270 // Catalog cleanup is async. Wait on procedure to finish up. 271 ProcedureTestingUtility.waitAllProcedures( 272 TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor()); 273 assertNull("Parent FN should be null", fnm.getFavoredNodes(regionA)); 274 assertNull("Parent FN should be null", fnm.getFavoredNodes(regionB)); 275 276 List<HRegionInfo> regions = admin.getTableRegions(tableName); 277 278 TEST_UTIL.deleteTable(tableName); 279 280 checkNoFNForDeletedTable(regions); 281 } 282 283 private void checkNoFNForDeletedTable(List<HRegionInfo> regions) { 284 for (HRegionInfo region : regions) { 285 LOG.info("Testing if FN data for " + region); 286 assertNull("FN not null for deleted table's region: " + region, fnm.getFavoredNodes(region)); 287 } 288 } 289 290 /* 291 * This checks the following: 292 * 293 * 1. Do all regions of the table have favored nodes updated in master? 294 * 2. Is the number of favored nodes correct for a region? Is the start code -1? 295 * 3. Is the FN information consistent between Master and the respective RegionServer? 296 */ 297 private void checkIfFavoredNodeInformationIsCorrect(TableName tableName) throws Exception { 298 299 /* 300 * Since we need HRegionServer to check for consistency of FN between Master and RS, 301 * lets construct a map for each serverName lookup. Makes it easy later. 302 */ 303 Map<ServerName, HRegionServer> snRSMap = Maps.newHashMap(); 304 for (JVMClusterUtil.RegionServerThread rst : 305 TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()) { 306 snRSMap.put(rst.getRegionServer().getServerName(), rst.getRegionServer()); 307 } 308 // Also include master, since it can also host user regions. 309 for (JVMClusterUtil.MasterThread rst : 310 TEST_UTIL.getMiniHBaseCluster().getLiveMasterThreads()) { 311 snRSMap.put(rst.getMaster().getServerName(), rst.getMaster()); 312 } 313 314 int dnPort = fnm.getDataNodePort(); 315 RegionLocator regionLocator = admin.getConnection().getRegionLocator(tableName); 316 for (HRegionLocation regionLocation : regionLocator.getAllRegionLocations()) { 317 318 HRegionInfo regionInfo = regionLocation.getRegionInfo(); 319 List<ServerName> fnList = fnm.getFavoredNodes(regionInfo); 320 321 // 1. Does each region have favored node? 322 assertNotNull("Favored nodes should not be null for region:" + regionInfo, fnList); 323 324 // 2. Do we have the right number of favored nodes? Is start code -1? 325 assertEquals("Incorrect favored nodes for region:" + regionInfo + " fnlist: " + fnList, 326 FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, fnList.size()); 327 for (ServerName sn : fnList) { 328 assertEquals("FN should not have startCode, fnlist:" + fnList, -1, sn.getStartcode()); 329 } 330 331 // 3. Check if the regionServers have all the FN updated and in sync with Master 332 HRegionServer regionServer = snRSMap.get(regionLocation.getServerName()); 333 assertNotNull("RS should not be null for regionLocation: " + regionLocation, regionServer); 334 335 InetSocketAddress[] rsFavNodes = 336 regionServer.getFavoredNodesForRegion(regionInfo.getEncodedName()); 337 assertNotNull("RS " + regionLocation.getServerName() 338 + " does not have FN for region: " + regionInfo, rsFavNodes); 339 assertEquals("Incorrect FN for region:" + regionInfo.getEncodedName() + " on server:" + 340 regionLocation.getServerName(), FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, 341 rsFavNodes.length); 342 343 // 4. Does DN port match all FN node list? 344 for (ServerName sn : fnm.getFavoredNodesWithDNPort(regionInfo)) { 345 assertEquals("FN should not have startCode, fnlist:" + fnList, -1, sn.getStartcode()); 346 assertEquals("FN port should belong to DN port, fnlist:" + fnList, dnPort, sn.getPort()); 347 } 348 } 349 } 350 351 /* 352 * Check favored nodes for system tables 353 */ 354 @Test 355 public void testSystemTables() throws Exception { 356 final TableName tableName = TableName.valueOf(name.getMethodName()); 357 TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys); 358 TEST_UTIL.waitUntilAllRegionsAssigned(tableName); 359 360 // All regions should have favored nodes 361 checkIfFavoredNodeInformationIsCorrect(tableName); 362 363 for (TableName sysTable : 364 admin.listTableNamesByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) { 365 List<HRegionInfo> regions = admin.getTableRegions(sysTable); 366 for (HRegionInfo region : regions) { 367 assertNull("FN should be null for sys region", fnm.getFavoredNodes(region)); 368 } 369 } 370 371 TEST_UTIL.deleteTable(tableName); 372 } 373 374 private void checkIfDaughterInherits2FN(List<ServerName> parentFN, List<ServerName> daughterFN) { 375 376 assertNotNull(parentFN); 377 assertNotNull(daughterFN); 378 379 List<ServerName> favoredNodes = Lists.newArrayList(daughterFN); 380 favoredNodes.removeAll(parentFN); 381 382 /* 383 * With a small cluster its likely some FN might accidentally get shared. Its likely the 384 * 3rd FN the balancer chooses might still belong to the parent in which case favoredNodes 385 * size would be 0. 386 */ 387 assertTrue("Daughter FN:" + daughterFN + " should have inherited 2 FN from parent FN:" 388 + parentFN, favoredNodes.size() <= 1); 389 } 390 391 private void waitUntilTableRegionCountReached(final TableName tableName, final int numRegions) 392 throws Exception { 393 TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { 394 @Override 395 public boolean evaluate() throws Exception { 396 return MetaTableAccessor.getRegionCount(TEST_UTIL.getConfiguration(), tableName) == numRegions; 397 } 398 }); 399 } 400}