001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.util; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022 023import java.io.File; 024import java.io.FileWriter; 025import java.io.IOException; 026import java.util.Collections; 027import java.util.List; 028 029import org.apache.hadoop.conf.Configuration; 030import org.apache.hadoop.fs.Path; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HBaseTestingUtility; 033import org.apache.hadoop.hbase.HConstants; 034import org.apache.hadoop.hbase.MiniHBaseCluster; 035import org.apache.hadoop.hbase.ServerName; 036import org.apache.hadoop.hbase.TableName; 037import org.apache.hadoop.hbase.Waiter.Predicate; 038import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 039import org.apache.hadoop.hbase.client.TableDescriptor; 040import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 041import org.apache.hadoop.hbase.regionserver.HRegion; 042import org.apache.hadoop.hbase.regionserver.HRegionServer; 043import org.apache.hadoop.hbase.testclassification.LargeTests; 044import org.apache.hadoop.hbase.testclassification.MiscTests; 045import org.apache.hadoop.hbase.util.RegionMover.RegionMoverBuilder; 046import org.junit.After; 047import org.junit.AfterClass; 048import org.junit.Assert; 049import org.junit.Before; 050import org.junit.BeforeClass; 051import org.junit.ClassRule; 052import org.junit.Rule; 053import org.junit.Test; 054import org.junit.experimental.categories.Category; 055import org.junit.rules.TestName; 056import org.slf4j.Logger; 057import org.slf4j.LoggerFactory; 058 059/** 060 * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test 061 * exclude functionality useful for rack decommissioning 062 */ 063@Category({MiscTests.class, LargeTests.class}) 064public class TestRegionMover1 { 065 066 @ClassRule 067 public static final HBaseClassTestRule CLASS_RULE = 068 HBaseClassTestRule.forClass(TestRegionMover1.class); 069 070 @Rule 071 public TestName name = new TestName(); 072 073 private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover1.class); 074 075 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 076 077 @BeforeClass 078 public static void setUpBeforeClass() throws Exception { 079 TEST_UTIL.startMiniCluster(3); 080 TEST_UTIL.getAdmin().balancerSwitch(false, true); 081 } 082 083 @AfterClass 084 public static void tearDownAfterClass() throws Exception { 085 TEST_UTIL.shutdownMiniCluster(); 086 } 087 088 @Before 089 public void setUp() throws Exception { 090 final TableName tableName = TableName.valueOf(name.getMethodName()); 091 TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName) 092 .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build(); 093 String startKey = "a"; 094 String endKey = "z"; 095 TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9); 096 } 097 098 @After 099 public void tearDown() throws Exception { 100 final TableName tableName = TableName.valueOf(name.getMethodName()); 101 TEST_UTIL.getAdmin().disableTable(tableName); 102 TEST_UTIL.getAdmin().deleteTable(tableName); 103 } 104 105 @Test 106 public void testWithAck() throws Exception { 107 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 108 HRegionServer regionServer = cluster.getRegionServer(0); 109 String rsName = regionServer.getServerName().getAddress().toString(); 110 int numRegions = regionServer.getNumberOfOnlineRegions(); 111 RegionMoverBuilder rmBuilder = 112 new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).maxthreads(8); 113 try (RegionMover rm = rmBuilder.build()) { 114 LOG.info("Unloading " + regionServer.getServerName()); 115 rm.unload(); 116 assertEquals(0, regionServer.getNumberOfOnlineRegions()); 117 LOG.info("Successfully Unloaded\nNow Loading"); 118 rm.load(); 119 assertEquals(numRegions, regionServer.getNumberOfOnlineRegions()); 120 // Repeat the same load. It should be very fast because all regions are already moved. 121 rm.load(); 122 } 123 } 124 125 /** 126 * Test to unload a regionserver first and then load it using no Ack mode. 127 */ 128 @Test 129 public void testWithoutAck() throws Exception { 130 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 131 HRegionServer regionServer = cluster.getRegionServer(0); 132 String rsName = regionServer.getServerName().getAddress().toString(); 133 int numRegions = regionServer.getNumberOfOnlineRegions(); 134 RegionMoverBuilder rmBuilder = 135 new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(false); 136 try (RegionMover rm = rmBuilder.build()) { 137 LOG.info("Unloading " + regionServer.getServerName()); 138 rm.unload(); 139 TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() { 140 @Override 141 public boolean evaluate() throws Exception { 142 return regionServer.getNumberOfOnlineRegions() == 0; 143 } 144 }); 145 LOG.info("Successfully Unloaded\nNow Loading"); 146 rm.load(); 147 // In UT we only have 10 regions so it is not likely to fail, so here we check for all 148 // regions, in the real production this may not be true. 149 TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() { 150 @Override 151 public boolean evaluate() throws Exception { 152 return regionServer.getNumberOfOnlineRegions() == numRegions; 153 } 154 }); 155 } 156 } 157 158 /** 159 * To test that we successfully exclude a server from the unloading process We test for the number 160 * of regions on Excluded server and also test that regions are unloaded successfully 161 */ 162 @Test 163 public void testExclude() throws Exception { 164 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 165 File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file"); 166 FileWriter fos = new FileWriter(excludeFile); 167 HRegionServer excludeServer = cluster.getRegionServer(1); 168 String excludeHostname = excludeServer.getServerName().getHostname(); 169 int excludeServerPort = excludeServer.getServerName().getPort(); 170 int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions(); 171 String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort); 172 fos.write(excludeServerName); 173 fos.close(); 174 HRegionServer regionServer = cluster.getRegionServer(0); 175 String rsName = regionServer.getServerName().getHostname(); 176 int port = regionServer.getServerName().getPort(); 177 String rs = rsName + ":" + Integer.toString(port); 178 RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration()) 179 .ack(true).excludeFile(excludeFile.getCanonicalPath()); 180 try (RegionMover rm = rmBuilder.build()) { 181 rm.unload(); 182 LOG.info("Unloading " + rs); 183 assertEquals(0, regionServer.getNumberOfOnlineRegions()); 184 assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions()); 185 LOG.info("Before:" + regionsExcludeServer + " After:" + 186 cluster.getRegionServer(1).getNumberOfOnlineRegions()); 187 } 188 } 189 190 @Test 191 public void testDesignatedFile() throws Exception{ 192 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 193 File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), 194 "designated_file"); 195 HRegionServer designatedServer = cluster.getRegionServer(0); 196 try(FileWriter fos = new FileWriter(designatedFile)) { 197 String designatedHostname = designatedServer.getServerName().getHostname(); 198 int designatedServerPort = designatedServer.getServerName().getPort(); 199 String excludeServerName = designatedHostname + ":" + designatedServerPort; 200 fos.write(excludeServerName); 201 } 202 int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions(); 203 HRegionServer regionServer = cluster.getRegionServer(1); 204 String rsName = regionServer.getServerName().getHostname(); 205 int port = regionServer.getServerName().getPort(); 206 String rs = rsName + ":" + port; 207 int regionsInRegionServer = regionServer.getNumberOfOnlineRegions(); 208 RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration()) 209 .designatedFile(designatedFile.getCanonicalPath()); 210 try (RegionMover rm = rmBuilder.build()) { 211 LOG.debug("Unloading {} regions", rs); 212 rm.unload(); 213 assertEquals(0, regionServer.getNumberOfOnlineRegions()); 214 assertEquals(regionsInDesignatedServer + regionsInRegionServer, 215 designatedServer.getNumberOfOnlineRegions()); 216 LOG.debug("Before:{} After:{}", regionsInDesignatedServer, 217 designatedServer.getNumberOfOnlineRegions()); 218 } 219 } 220 221 @Test 222 public void testExcludeAndDesignated() throws Exception{ 223 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 224 // create designated file 225 File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), 226 "designated_file"); 227 HRegionServer designatedServer = cluster.getRegionServer(0); 228 try(FileWriter fos = new FileWriter(designatedFile)) { 229 String designatedHostname = designatedServer.getServerName().getHostname(); 230 int designatedServerPort = designatedServer.getServerName().getPort(); 231 String excludeServerName = designatedHostname + ":" + designatedServerPort; 232 fos.write(excludeServerName); 233 } 234 int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions(); 235 // create exclude file 236 File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file"); 237 HRegionServer excludeServer = cluster.getRegionServer(1); 238 try(FileWriter fos = new FileWriter(excludeFile)) { 239 String excludeHostname = excludeServer.getServerName().getHostname(); 240 int excludeServerPort = excludeServer.getServerName().getPort(); 241 String excludeServerName = excludeHostname + ":" + excludeServerPort; 242 fos.write(excludeServerName); 243 } 244 int regionsInExcludeServer = excludeServer.getNumberOfOnlineRegions(); 245 246 HRegionServer targetRegionServer = cluster.getRegionServer(2); 247 String rsName = targetRegionServer.getServerName().getHostname(); 248 int port = targetRegionServer.getServerName().getPort(); 249 String rs = rsName + ":" + port; 250 int regionsInTargetRegionServer = targetRegionServer.getNumberOfOnlineRegions(); 251 252 RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration()) 253 .designatedFile(designatedFile.getCanonicalPath()) 254 .excludeFile(excludeFile.getCanonicalPath()); 255 try (RegionMover rm = rmBuilder.build()) { 256 LOG.debug("Unloading {}", rs); 257 rm.unload(); 258 assertEquals(0, targetRegionServer.getNumberOfOnlineRegions()); 259 assertEquals(regionsInDesignatedServer + regionsInTargetRegionServer, 260 designatedServer.getNumberOfOnlineRegions()); 261 LOG.debug("DesignatedServer Before:{} After:{}", regionsInDesignatedServer, 262 designatedServer.getNumberOfOnlineRegions()); 263 assertEquals(regionsInExcludeServer, excludeServer.getNumberOfOnlineRegions()); 264 LOG.debug("ExcludeServer Before:{} After:{}", regionsInExcludeServer, 265 excludeServer.getNumberOfOnlineRegions()); 266 } 267 } 268 269 @Test 270 public void testRegionServerPort() throws Exception { 271 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 272 HRegionServer regionServer = cluster.getRegionServer(0); 273 String rsName = regionServer.getServerName().getHostname(); 274 275 final int PORT = 16021; 276 Configuration conf = TEST_UTIL.getConfiguration(); 277 String originalPort = conf.get(HConstants.REGIONSERVER_PORT); 278 conf.set(HConstants.REGIONSERVER_PORT, Integer.toString(PORT)); 279 RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rsName, conf); 280 assertEquals(PORT, rmBuilder.port); 281 if (originalPort != null) { 282 conf.set(HConstants.REGIONSERVER_PORT, originalPort); 283 } 284 } 285 286 /** 287 * UT for HBASE-21746 288 */ 289 @Test 290 public void testLoadMetaRegion() throws Exception { 291 HRegionServer rsWithMeta = TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream() 292 .map(t -> t.getRegionServer()) 293 .filter(rs -> rs.getRegions(TableName.META_TABLE_NAME).size() > 0).findFirst().get(); 294 int onlineRegions = rsWithMeta.getNumberOfOnlineRegions(); 295 String rsName = rsWithMeta.getServerName().getAddress().toString(); 296 try (RegionMover rm = 297 new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).build()) { 298 LOG.info("Unloading " + rsWithMeta.getServerName()); 299 rm.unload(); 300 assertEquals(0, rsWithMeta.getNumberOfOnlineRegions()); 301 LOG.info("Loading " + rsWithMeta.getServerName()); 302 rm.load(); 303 assertEquals(onlineRegions, rsWithMeta.getNumberOfOnlineRegions()); 304 } 305 } 306 307 /** 308 * UT for HBASE-21746 309 */ 310 @Test 311 public void testTargetServerDeadWhenLoading() throws Exception { 312 HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); 313 String rsName = rs.getServerName().getAddress().toString(); 314 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); 315 // wait 5 seconds at most 316 conf.setInt(RegionMover.SERVERSTART_WAIT_MAX_KEY, 5); 317 String filename = 318 new Path(TEST_UTIL.getDataTestDir(), "testTargetServerDeadWhenLoading").toString(); 319 // unload the region server 320 try (RegionMover rm = 321 new RegionMoverBuilder(rsName, conf).filename(filename).ack(true).build()) { 322 LOG.info("Unloading " + rs.getServerName()); 323 rm.unload(); 324 assertEquals(0, rs.getNumberOfOnlineRegions()); 325 } 326 String inexistRsName = "whatever:123"; 327 try (RegionMover rm = 328 new RegionMoverBuilder(inexistRsName, conf).filename(filename).ack(true).build()) { 329 // load the regions to an inexist region server, which should fail and return false 330 LOG.info("Loading to an inexist region server {}", inexistRsName); 331 assertFalse(rm.load()); 332 } 333 } 334 335 @Test 336 public void testDecomServerExclusionWithAck() throws Exception { 337 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 338 HRegionServer excludeServer = cluster.getRegionServer(1); 339 List<HRegion> regions = excludeServer.getRegions(); 340 int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions(); 341 TEST_UTIL.getAdmin().decommissionRegionServers( 342 Collections.singletonList(excludeServer.getServerName()), false); 343 344 waitForServerDecom(excludeServer); 345 346 HRegionServer regionServer = cluster.getRegionServer(0); 347 String rsName = regionServer.getServerName().getHostname(); 348 int port = regionServer.getServerName().getPort(); 349 String hostname = rsName + ":" + Integer.toString(port); 350 RegionMoverBuilder rmBuilder = 351 new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration()) 352 .ack(true); 353 354 int targetServerRegions = cluster.getRegionServer(2).getRegions().size(); 355 int sourceServerRegions = regionServer.getRegions().size(); 356 357 try (RegionMover regionMover = rmBuilder.build()) { 358 Assert.assertTrue(regionMover.unload()); 359 LOG.info("Unloading {}", hostname); 360 assertEquals(0, regionServer.getNumberOfOnlineRegions()); 361 assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions()); 362 LOG.info("Before:" + regionsExcludeServer + " After:" + 363 cluster.getRegionServer(1).getNumberOfOnlineRegions()); 364 List<HRegion> regionList = cluster.getRegionServer(1).getRegions(); 365 int index = 0; 366 for (HRegion hRegion : regionList) { 367 Assert.assertEquals(hRegion, regions.get(index++)); 368 } 369 Assert.assertEquals(targetServerRegions + sourceServerRegions, 370 cluster.getRegionServer(2).getNumberOfOnlineRegions()); 371 Assert.assertTrue(regionMover.load()); 372 } 373 374 TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(), 375 Collections.emptyList()); 376 } 377 378 private void waitForServerDecom(HRegionServer excludeServer) { 379 380 TEST_UTIL.waitFor(3000, () -> { 381 try { 382 List<ServerName> decomServers = TEST_UTIL.getAdmin().listDecommissionedRegionServers(); 383 return decomServers.size() == 1 384 && decomServers.get(0).equals(excludeServer.getServerName()); 385 } catch (IOException e) { 386 throw new RuntimeException(e); 387 } 388 }); 389 } 390 391 @Test 392 public void testDecomServerExclusion() throws Exception { 393 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 394 HRegionServer excludeServer = cluster.getRegionServer(0); 395 List<HRegion> regions = excludeServer.getRegions(); 396 int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions(); 397 TEST_UTIL.getAdmin().decommissionRegionServers( 398 Collections.singletonList(excludeServer.getServerName()), false); 399 400 waitForServerDecom(excludeServer); 401 402 HRegionServer sourceRegionServer = cluster.getRegionServer(1); 403 String rsName = sourceRegionServer.getServerName().getHostname(); 404 int port = sourceRegionServer.getServerName().getPort(); 405 String hostname = rsName + ":" + Integer.toString(port); 406 RegionMoverBuilder rmBuilder = 407 new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration()).ack(false); 408 409 int targetServerRegions = cluster.getRegionServer(2).getRegions().size(); 410 int sourceServerRegions = sourceRegionServer.getRegions().size(); 411 412 try (RegionMover regionMover = rmBuilder.build()) { 413 Assert.assertTrue(regionMover.unload()); 414 LOG.info("Unloading {}", hostname); 415 assertEquals(0, sourceRegionServer.getNumberOfOnlineRegions()); 416 assertEquals(regionsExcludeServer, cluster.getRegionServer(0).getNumberOfOnlineRegions()); 417 LOG.info("Before:" + regionsExcludeServer + " After:" + 418 cluster.getRegionServer(1).getNumberOfOnlineRegions()); 419 List<HRegion> regionList = cluster.getRegionServer(0).getRegions(); 420 int index = 0; 421 for (HRegion hRegion : regionList) { 422 Assert.assertEquals(hRegion, regions.get(index++)); 423 } 424 Assert.assertEquals(targetServerRegions + sourceServerRegions, 425 cluster.getRegionServer(2).getNumberOfOnlineRegions()); 426 Assert.assertTrue(regionMover.load()); 427 } 428 429 TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(), 430 Collections.emptyList()); 431 } 432 433 @Test 434 public void testExcludeAndDecomServers() throws Exception { 435 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 436 File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file"); 437 FileWriter fos = new FileWriter(excludeFile); 438 HRegionServer excludeServer = cluster.getRegionServer(1); 439 String excludeHostname = excludeServer.getServerName().getHostname(); 440 int excludeServerPort = excludeServer.getServerName().getPort(); 441 String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort); 442 fos.write(excludeServerName); 443 fos.close(); 444 445 HRegionServer decomServer = cluster.getRegionServer(2); 446 TEST_UTIL.getAdmin().decommissionRegionServers( 447 Collections.singletonList(decomServer.getServerName()), false); 448 449 waitForServerDecom(decomServer); 450 451 HRegionServer regionServer = cluster.getRegionServer(0); 452 String rsName = regionServer.getServerName().getHostname(); 453 int port = regionServer.getServerName().getPort(); 454 String sourceServer = rsName + ":" + Integer.toString(port); 455 RegionMoverBuilder rmBuilder = 456 new RegionMoverBuilder(sourceServer, TEST_UTIL.getConfiguration()) 457 .ack(true) 458 .excludeFile(excludeFile.getCanonicalPath()); 459 try (RegionMover regionMover = rmBuilder.build()) { 460 Assert.assertFalse(regionMover.unload()); 461 } 462 463 TEST_UTIL.getAdmin().recommissionRegionServer(decomServer.getServerName(), 464 Collections.emptyList()); 465 } 466 467}