001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.util; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022 023import java.io.File; 024import java.io.FileWriter; 025import java.io.IOException; 026import java.util.Collections; 027import java.util.List; 028 029import org.apache.hadoop.conf.Configuration; 030import org.apache.hadoop.fs.Path; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HBaseTestingUtility; 033import org.apache.hadoop.hbase.HConstants; 034import org.apache.hadoop.hbase.MiniHBaseCluster; 035import org.apache.hadoop.hbase.ServerName; 036import org.apache.hadoop.hbase.TableName; 037import org.apache.hadoop.hbase.Waiter.Predicate; 038import org.apache.hadoop.hbase.client.Admin; 039import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 040import org.apache.hadoop.hbase.client.TableDescriptor; 041import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 042import org.apache.hadoop.hbase.regionserver.HRegion; 043import org.apache.hadoop.hbase.regionserver.HRegionServer; 044import org.apache.hadoop.hbase.testclassification.LargeTests; 045import org.apache.hadoop.hbase.testclassification.MiscTests; 046import org.apache.hadoop.hbase.util.RegionMover.RegionMoverBuilder; 047import org.junit.AfterClass; 048import org.junit.Assert; 049import org.junit.Before; 050import org.junit.BeforeClass; 051import org.junit.ClassRule; 052import org.junit.Test; 053import org.junit.experimental.categories.Category; 054import org.slf4j.Logger; 055import org.slf4j.LoggerFactory; 056 057/** 058 * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test 059 * exclude functionality useful for rack decommissioning 060 */ 061@Category({MiscTests.class, LargeTests.class}) 062public class TestRegionMover { 063 064 @ClassRule 065 public static final HBaseClassTestRule CLASS_RULE = 066 HBaseClassTestRule.forClass(TestRegionMover.class); 067 068 private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover.class); 069 070 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 071 072 @BeforeClass 073 public static void setUpBeforeClass() throws Exception { 074 TEST_UTIL.startMiniCluster(3); 075 TEST_UTIL.getAdmin().balancerSwitch(false, true); 076 } 077 078 @AfterClass 079 public static void tearDownAfterClass() throws Exception { 080 TEST_UTIL.shutdownMiniCluster(); 081 } 082 083 @Before 084 public void setUp() throws Exception { 085 // Create a pre-split table just to populate some regions 086 TableName tableName = TableName.valueOf("testRegionMover"); 087 Admin admin = TEST_UTIL.getAdmin(); 088 if (admin.tableExists(tableName)) { 089 TEST_UTIL.deleteTable(tableName); 090 } 091 TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName) 092 .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build(); 093 String startKey = "a"; 094 String endKey = "z"; 095 admin.createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9); 096 } 097 098 @Test 099 public void testWithAck() throws Exception { 100 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 101 HRegionServer regionServer = cluster.getRegionServer(0); 102 String rsName = regionServer.getServerName().getAddress().toString(); 103 int numRegions = regionServer.getNumberOfOnlineRegions(); 104 RegionMoverBuilder rmBuilder = 105 new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).maxthreads(8); 106 try (RegionMover rm = rmBuilder.build()) { 107 LOG.info("Unloading " + regionServer.getServerName()); 108 rm.unload(); 109 assertEquals(0, regionServer.getNumberOfOnlineRegions()); 110 LOG.info("Successfully Unloaded\nNow Loading"); 111 rm.load(); 112 assertEquals(numRegions, regionServer.getNumberOfOnlineRegions()); 113 // Repeat the same load. It should be very fast because all regions are already moved. 114 rm.load(); 115 } 116 } 117 118 /** 119 * Test to unload a regionserver first and then load it using no Ack mode. 120 */ 121 @Test 122 public void testWithoutAck() throws Exception { 123 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 124 HRegionServer regionServer = cluster.getRegionServer(0); 125 String rsName = regionServer.getServerName().getAddress().toString(); 126 int numRegions = regionServer.getNumberOfOnlineRegions(); 127 RegionMoverBuilder rmBuilder = 128 new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(false); 129 try (RegionMover rm = rmBuilder.build()) { 130 LOG.info("Unloading " + regionServer.getServerName()); 131 rm.unload(); 132 TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() { 133 @Override 134 public boolean evaluate() throws Exception { 135 return regionServer.getNumberOfOnlineRegions() == 0; 136 } 137 }); 138 LOG.info("Successfully Unloaded\nNow Loading"); 139 rm.load(); 140 // In UT we only have 10 regions so it is not likely to fail, so here we check for all 141 // regions, in the real production this may not be true. 142 TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() { 143 @Override 144 public boolean evaluate() throws Exception { 145 return regionServer.getNumberOfOnlineRegions() == numRegions; 146 } 147 }); 148 } 149 } 150 151 /** 152 * To test that we successfully exclude a server from the unloading process We test for the number 153 * of regions on Excluded server and also test that regions are unloaded successfully 154 */ 155 @Test 156 public void testExclude() throws Exception { 157 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 158 File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file"); 159 FileWriter fos = new FileWriter(excludeFile); 160 HRegionServer excludeServer = cluster.getRegionServer(1); 161 String excludeHostname = excludeServer.getServerName().getHostname(); 162 int excludeServerPort = excludeServer.getServerName().getPort(); 163 int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions(); 164 String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort); 165 fos.write(excludeServerName); 166 fos.close(); 167 HRegionServer regionServer = cluster.getRegionServer(0); 168 String rsName = regionServer.getServerName().getHostname(); 169 int port = regionServer.getServerName().getPort(); 170 String rs = rsName + ":" + Integer.toString(port); 171 RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration()) 172 .ack(true).excludeFile(excludeFile.getCanonicalPath()); 173 try (RegionMover rm = rmBuilder.build()) { 174 rm.unload(); 175 LOG.info("Unloading " + rs); 176 assertEquals(0, regionServer.getNumberOfOnlineRegions()); 177 assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions()); 178 LOG.info("Before:" + regionsExcludeServer + " After:" + 179 cluster.getRegionServer(1).getNumberOfOnlineRegions()); 180 } 181 } 182 183 @Test 184 public void testDesignatedFile() throws Exception{ 185 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 186 File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), 187 "designated_file"); 188 HRegionServer designatedServer = cluster.getRegionServer(0); 189 try(FileWriter fos = new FileWriter(designatedFile)) { 190 String designatedHostname = designatedServer.getServerName().getHostname(); 191 int designatedServerPort = designatedServer.getServerName().getPort(); 192 String excludeServerName = designatedHostname + ":" + designatedServerPort; 193 fos.write(excludeServerName); 194 } 195 int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions(); 196 HRegionServer regionServer = cluster.getRegionServer(1); 197 String rsName = regionServer.getServerName().getHostname(); 198 int port = regionServer.getServerName().getPort(); 199 String rs = rsName + ":" + port; 200 int regionsInRegionServer = regionServer.getNumberOfOnlineRegions(); 201 RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration()) 202 .designatedFile(designatedFile.getCanonicalPath()); 203 try (RegionMover rm = rmBuilder.build()) { 204 LOG.debug("Unloading {} regions", rs); 205 rm.unload(); 206 assertEquals(0, regionServer.getNumberOfOnlineRegions()); 207 assertEquals(regionsInDesignatedServer + regionsInRegionServer, 208 designatedServer.getNumberOfOnlineRegions()); 209 LOG.debug("Before:{} After:{}", regionsInDesignatedServer, 210 designatedServer.getNumberOfOnlineRegions()); 211 } 212 } 213 214 @Test 215 public void testExcludeAndDesignated() throws Exception{ 216 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 217 // create designated file 218 File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), 219 "designated_file"); 220 HRegionServer designatedServer = cluster.getRegionServer(0); 221 try(FileWriter fos = new FileWriter(designatedFile)) { 222 String designatedHostname = designatedServer.getServerName().getHostname(); 223 int designatedServerPort = designatedServer.getServerName().getPort(); 224 String excludeServerName = designatedHostname + ":" + designatedServerPort; 225 fos.write(excludeServerName); 226 } 227 int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions(); 228 // create exclude file 229 File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file"); 230 HRegionServer excludeServer = cluster.getRegionServer(1); 231 try(FileWriter fos = new FileWriter(excludeFile)) { 232 String excludeHostname = excludeServer.getServerName().getHostname(); 233 int excludeServerPort = excludeServer.getServerName().getPort(); 234 String excludeServerName = excludeHostname + ":" + excludeServerPort; 235 fos.write(excludeServerName); 236 } 237 int regionsInExcludeServer = excludeServer.getNumberOfOnlineRegions(); 238 239 HRegionServer targetRegionServer = cluster.getRegionServer(2); 240 String rsName = targetRegionServer.getServerName().getHostname(); 241 int port = targetRegionServer.getServerName().getPort(); 242 String rs = rsName + ":" + port; 243 int regionsInTargetRegionServer = targetRegionServer.getNumberOfOnlineRegions(); 244 245 RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration()) 246 .designatedFile(designatedFile.getCanonicalPath()) 247 .excludeFile(excludeFile.getCanonicalPath()); 248 try (RegionMover rm = rmBuilder.build()) { 249 LOG.debug("Unloading {}", rs); 250 rm.unload(); 251 assertEquals(0, targetRegionServer.getNumberOfOnlineRegions()); 252 assertEquals(regionsInDesignatedServer + regionsInTargetRegionServer, 253 designatedServer.getNumberOfOnlineRegions()); 254 LOG.debug("DesignatedServer Before:{} After:{}", regionsInDesignatedServer, 255 designatedServer.getNumberOfOnlineRegions()); 256 assertEquals(regionsInExcludeServer, excludeServer.getNumberOfOnlineRegions()); 257 LOG.debug("ExcludeServer Before:{} After:{}", regionsInExcludeServer, 258 excludeServer.getNumberOfOnlineRegions()); 259 } 260 } 261 262 @Test 263 public void testRegionServerPort() { 264 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 265 HRegionServer regionServer = cluster.getRegionServer(0); 266 String rsName = regionServer.getServerName().getHostname(); 267 268 final int PORT = 16021; 269 Configuration conf = TEST_UTIL.getConfiguration(); 270 String originalPort = conf.get(HConstants.REGIONSERVER_PORT); 271 conf.set(HConstants.REGIONSERVER_PORT, Integer.toString(PORT)); 272 RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rsName, conf); 273 assertEquals(PORT, rmBuilder.port); 274 if (originalPort != null) { 275 conf.set(HConstants.REGIONSERVER_PORT, originalPort); 276 } 277 } 278 279 /** 280 * UT for HBASE-21746 281 */ 282 @Test 283 public void testLoadMetaRegion() throws Exception { 284 HRegionServer rsWithMeta = TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream() 285 .map(t -> t.getRegionServer()) 286 .filter(rs -> rs.getRegions(TableName.META_TABLE_NAME).size() > 0).findFirst().get(); 287 int onlineRegions = rsWithMeta.getNumberOfOnlineRegions(); 288 String rsName = rsWithMeta.getServerName().getAddress().toString(); 289 try (RegionMover rm = 290 new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).build()) { 291 LOG.info("Unloading " + rsWithMeta.getServerName()); 292 rm.unload(); 293 assertEquals(0, rsWithMeta.getNumberOfOnlineRegions()); 294 LOG.info("Loading " + rsWithMeta.getServerName()); 295 rm.load(); 296 assertEquals(onlineRegions, rsWithMeta.getNumberOfOnlineRegions()); 297 } 298 } 299 300 /** 301 * UT for HBASE-21746 302 */ 303 @Test 304 public void testTargetServerDeadWhenLoading() throws Exception { 305 HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); 306 String rsName = rs.getServerName().getAddress().toString(); 307 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); 308 // wait 5 seconds at most 309 conf.setInt(RegionMover.SERVERSTART_WAIT_MAX_KEY, 5); 310 String filename = 311 new Path(TEST_UTIL.getDataTestDir(), "testTargetServerDeadWhenLoading").toString(); 312 // unload the region server 313 try (RegionMover rm = 314 new RegionMoverBuilder(rsName, conf).filename(filename).ack(true).build()) { 315 LOG.info("Unloading " + rs.getServerName()); 316 rm.unload(); 317 assertEquals(0, rs.getNumberOfOnlineRegions()); 318 } 319 String inexistRsName = "whatever:123"; 320 try (RegionMover rm = 321 new RegionMoverBuilder(inexistRsName, conf).filename(filename).ack(true).build()) { 322 // load the regions to an inexist region server, which should fail and return false 323 LOG.info("Loading to an inexist region server {}", inexistRsName); 324 assertFalse(rm.load()); 325 } 326 } 327 328 @Test 329 public void testDecomServerExclusionWithAck() throws Exception { 330 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 331 HRegionServer excludeServer = cluster.getRegionServer(1); 332 List<HRegion> regions = excludeServer.getRegions(); 333 int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions(); 334 TEST_UTIL.getAdmin().decommissionRegionServers( 335 Collections.singletonList(excludeServer.getServerName()), false); 336 337 waitForServerDecom(excludeServer); 338 339 HRegionServer regionServer = cluster.getRegionServer(0); 340 String rsName = regionServer.getServerName().getHostname(); 341 int port = regionServer.getServerName().getPort(); 342 String hostname = rsName + ":" + Integer.toString(port); 343 RegionMoverBuilder rmBuilder = 344 new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration()) 345 .ack(true); 346 347 int targetServerRegions = cluster.getRegionServer(2).getRegions().size(); 348 int sourceServerRegions = regionServer.getRegions().size(); 349 350 try (RegionMover regionMover = rmBuilder.build()) { 351 Assert.assertTrue(regionMover.unload()); 352 LOG.info("Unloading {}", hostname); 353 assertEquals(0, regionServer.getNumberOfOnlineRegions()); 354 assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions()); 355 LOG.info("Before:" + regionsExcludeServer + " After:" + 356 cluster.getRegionServer(1).getNumberOfOnlineRegions()); 357 List<HRegion> regionList = cluster.getRegionServer(1).getRegions(); 358 int index = 0; 359 for (HRegion hRegion : regionList) { 360 Assert.assertEquals(hRegion, regions.get(index++)); 361 } 362 Assert.assertEquals(targetServerRegions + sourceServerRegions, 363 cluster.getRegionServer(2).getNumberOfOnlineRegions()); 364 Assert.assertTrue(regionMover.load()); 365 } 366 367 TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(), 368 Collections.emptyList()); 369 } 370 371 private void waitForServerDecom(HRegionServer excludeServer) { 372 373 TEST_UTIL.waitFor(3000, () -> { 374 try { 375 List<ServerName> decomServers = TEST_UTIL.getAdmin().listDecommissionedRegionServers(); 376 return decomServers.size() == 1 377 && decomServers.get(0).equals(excludeServer.getServerName()); 378 } catch (IOException e) { 379 throw new RuntimeException(e); 380 } 381 }); 382 } 383 384 @Test 385 public void testDecomServerExclusion() throws Exception { 386 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 387 HRegionServer excludeServer = cluster.getRegionServer(0); 388 List<HRegion> regions = excludeServer.getRegions(); 389 int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions(); 390 TEST_UTIL.getAdmin().decommissionRegionServers( 391 Collections.singletonList(excludeServer.getServerName()), false); 392 393 waitForServerDecom(excludeServer); 394 395 HRegionServer sourceRegionServer = cluster.getRegionServer(1); 396 String rsName = sourceRegionServer.getServerName().getHostname(); 397 int port = sourceRegionServer.getServerName().getPort(); 398 String hostname = rsName + ":" + Integer.toString(port); 399 RegionMoverBuilder rmBuilder = 400 new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration()).ack(false); 401 402 int targetServerRegions = cluster.getRegionServer(2).getRegions().size(); 403 int sourceServerRegions = sourceRegionServer.getRegions().size(); 404 405 try (RegionMover regionMover = rmBuilder.build()) { 406 Assert.assertTrue(regionMover.unload()); 407 LOG.info("Unloading {}", hostname); 408 assertEquals(0, sourceRegionServer.getNumberOfOnlineRegions()); 409 assertEquals(regionsExcludeServer, cluster.getRegionServer(0).getNumberOfOnlineRegions()); 410 LOG.info("Before:" + regionsExcludeServer + " After:" + 411 cluster.getRegionServer(1).getNumberOfOnlineRegions()); 412 List<HRegion> regionList = cluster.getRegionServer(0).getRegions(); 413 int index = 0; 414 for (HRegion hRegion : regionList) { 415 Assert.assertEquals(hRegion, regions.get(index++)); 416 } 417 Assert.assertEquals(targetServerRegions + sourceServerRegions, 418 cluster.getRegionServer(2).getNumberOfOnlineRegions()); 419 Assert.assertTrue(regionMover.load()); 420 } 421 422 TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(), 423 Collections.emptyList()); 424 } 425 426 @Test 427 public void testExcludeAndDecomServers() throws Exception { 428 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 429 File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file"); 430 FileWriter fos = new FileWriter(excludeFile); 431 HRegionServer excludeServer = cluster.getRegionServer(1); 432 String excludeHostname = excludeServer.getServerName().getHostname(); 433 int excludeServerPort = excludeServer.getServerName().getPort(); 434 String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort); 435 fos.write(excludeServerName); 436 fos.close(); 437 438 HRegionServer decomServer = cluster.getRegionServer(2); 439 TEST_UTIL.getAdmin().decommissionRegionServers( 440 Collections.singletonList(decomServer.getServerName()), false); 441 442 waitForServerDecom(decomServer); 443 444 HRegionServer regionServer = cluster.getRegionServer(0); 445 String rsName = regionServer.getServerName().getHostname(); 446 int port = regionServer.getServerName().getPort(); 447 String sourceServer = rsName + ":" + Integer.toString(port); 448 RegionMoverBuilder rmBuilder = 449 new RegionMoverBuilder(sourceServer, TEST_UTIL.getConfiguration()) 450 .ack(true) 451 .excludeFile(excludeFile.getCanonicalPath()); 452 try (RegionMover regionMover = rmBuilder.build()) { 453 Assert.assertFalse(regionMover.unload()); 454 } 455 456 TEST_UTIL.getAdmin().recommissionRegionServer(decomServer.getServerName(), 457 Collections.emptyList()); 458 } 459 460}