001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.util; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022 023import java.io.File; 024import java.io.FileWriter; 025import java.io.IOException; 026import java.util.Collections; 027import java.util.List; 028 029import org.apache.hadoop.conf.Configuration; 030import org.apache.hadoop.fs.Path; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HBaseTestingUtility; 033import org.apache.hadoop.hbase.HConstants; 034import org.apache.hadoop.hbase.MiniHBaseCluster; 035import org.apache.hadoop.hbase.ServerName; 036import org.apache.hadoop.hbase.TableName; 037import org.apache.hadoop.hbase.Waiter.Predicate; 038import org.apache.hadoop.hbase.client.Admin; 039import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 040import org.apache.hadoop.hbase.client.TableDescriptor; 041import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 042import org.apache.hadoop.hbase.regionserver.HRegion; 043import org.apache.hadoop.hbase.regionserver.HRegionServer; 044import org.apache.hadoop.hbase.testclassification.LargeTests; 045import org.apache.hadoop.hbase.testclassification.MiscTests; 046import org.apache.hadoop.hbase.util.RegionMover.RegionMoverBuilder; 047import org.junit.AfterClass; 048import org.junit.Assert; 049import org.junit.Before; 050import org.junit.BeforeClass; 051import org.junit.ClassRule; 052import org.junit.Test; 053import org.junit.experimental.categories.Category; 054import org.slf4j.Logger; 055import org.slf4j.LoggerFactory; 056 057/** 058 * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test 059 * exclude functionality useful for rack decommissioning 060 */ 061@Category({MiscTests.class, LargeTests.class}) 062public class TestRegionMover { 063 064 @ClassRule 065 public static final HBaseClassTestRule CLASS_RULE = 066 HBaseClassTestRule.forClass(TestRegionMover.class); 067 068 private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover.class); 069 070 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 071 072 @BeforeClass 073 public static void setUpBeforeClass() throws Exception { 074 TEST_UTIL.startMiniCluster(3); 075 TEST_UTIL.getAdmin().balancerSwitch(false, true); 076 } 077 078 @AfterClass 079 public static void tearDownAfterClass() throws Exception { 080 TEST_UTIL.shutdownMiniCluster(); 081 } 082 083 @Before 084 public void setUp() throws Exception { 085 // Create a pre-split table just to populate some regions 086 TableName tableName = TableName.valueOf("testRegionMover"); 087 Admin admin = TEST_UTIL.getAdmin(); 088 if (admin.tableExists(tableName)) { 089 TEST_UTIL.deleteTable(tableName); 090 } 091 TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName) 092 .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build(); 093 String startKey = "a"; 094 String endKey = "z"; 095 admin.createTable(tableDesc, startKey.getBytes(), endKey.getBytes(), 9); 096 } 097 098 @Test 099 public void testWithAck() throws Exception { 100 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 101 HRegionServer regionServer = cluster.getRegionServer(0); 102 String rsName = regionServer.getServerName().getAddress().toString(); 103 int numRegions = regionServer.getNumberOfOnlineRegions(); 104 RegionMoverBuilder rmBuilder = 105 new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).maxthreads(8); 106 try (RegionMover rm = rmBuilder.build()) { 107 LOG.info("Unloading " + regionServer.getServerName()); 108 rm.unload(); 109 assertEquals(0, regionServer.getNumberOfOnlineRegions()); 110 LOG.info("Successfully Unloaded\nNow Loading"); 111 rm.load(); 112 assertEquals(numRegions, regionServer.getNumberOfOnlineRegions()); 113 // Repeat the same load. It should be very fast because all regions are already moved. 114 rm.load(); 115 } 116 } 117 118 /** 119 * Test to unload a regionserver first and then load it using no Ack mode. 120 */ 121 @Test 122 public void testWithoutAck() throws Exception { 123 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 124 HRegionServer regionServer = cluster.getRegionServer(0); 125 String rsName = regionServer.getServerName().getAddress().toString(); 126 int numRegions = regionServer.getNumberOfOnlineRegions(); 127 RegionMoverBuilder rmBuilder = 128 new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(false); 129 try (RegionMover rm = rmBuilder.build()) { 130 LOG.info("Unloading " + regionServer.getServerName()); 131 rm.unload(); 132 TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() { 133 @Override 134 public boolean evaluate() throws Exception { 135 return regionServer.getNumberOfOnlineRegions() == 0; 136 } 137 }); 138 LOG.info("Successfully Unloaded\nNow Loading"); 139 rm.load(); 140 // In UT we only have 10 regions so it is not likely to fail, so here we check for all 141 // regions, in the real production this may not be true. 142 TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() { 143 @Override 144 public boolean evaluate() throws Exception { 145 return regionServer.getNumberOfOnlineRegions() == numRegions; 146 } 147 }); 148 } 149 } 150 151 /** 152 * To test that we successfully exclude a server from the unloading process We test for the number 153 * of regions on Excluded server and also test that regions are unloaded successfully 154 */ 155 @Test 156 public void testExclude() throws Exception { 157 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 158 File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file"); 159 FileWriter fos = new FileWriter(excludeFile); 160 HRegionServer excludeServer = cluster.getRegionServer(1); 161 String excludeHostname = excludeServer.getServerName().getHostname(); 162 int excludeServerPort = excludeServer.getServerName().getPort(); 163 int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions(); 164 String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort); 165 fos.write(excludeServerName); 166 fos.close(); 167 HRegionServer regionServer = cluster.getRegionServer(0); 168 String rsName = regionServer.getServerName().getHostname(); 169 int port = regionServer.getServerName().getPort(); 170 String rs = rsName + ":" + Integer.toString(port); 171 RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration()) 172 .ack(true).excludeFile(excludeFile.getCanonicalPath()); 173 try (RegionMover rm = rmBuilder.build()) { 174 rm.unload(); 175 LOG.info("Unloading " + rs); 176 assertEquals(0, regionServer.getNumberOfOnlineRegions()); 177 assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions()); 178 LOG.info("Before:" + regionsExcludeServer + " After:" + 179 cluster.getRegionServer(1).getNumberOfOnlineRegions()); 180 } 181 } 182 183 @Test 184 public void testRegionServerPort() { 185 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 186 HRegionServer regionServer = cluster.getRegionServer(0); 187 String rsName = regionServer.getServerName().getHostname(); 188 189 final int PORT = 16021; 190 Configuration conf = TEST_UTIL.getConfiguration(); 191 String originalPort = conf.get(HConstants.REGIONSERVER_PORT); 192 conf.set(HConstants.REGIONSERVER_PORT, Integer.toString(PORT)); 193 RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rsName, conf); 194 assertEquals(PORT, rmBuilder.port); 195 if (originalPort != null) { 196 conf.set(HConstants.REGIONSERVER_PORT, originalPort); 197 } 198 } 199 200 /** 201 * UT for HBASE-21746 202 */ 203 @Test 204 public void testLoadMetaRegion() throws Exception { 205 HRegionServer rsWithMeta = TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream() 206 .map(t -> t.getRegionServer()) 207 .filter(rs -> rs.getRegions(TableName.META_TABLE_NAME).size() > 0).findFirst().get(); 208 int onlineRegions = rsWithMeta.getNumberOfOnlineRegions(); 209 String rsName = rsWithMeta.getServerName().getAddress().toString(); 210 try (RegionMover rm = 211 new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).build()) { 212 LOG.info("Unloading " + rsWithMeta.getServerName()); 213 rm.unload(); 214 assertEquals(0, rsWithMeta.getNumberOfOnlineRegions()); 215 LOG.info("Loading " + rsWithMeta.getServerName()); 216 rm.load(); 217 assertEquals(onlineRegions, rsWithMeta.getNumberOfOnlineRegions()); 218 } 219 } 220 221 /** 222 * UT for HBASE-21746 223 */ 224 @Test 225 public void testTargetServerDeadWhenLoading() throws Exception { 226 HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); 227 String rsName = rs.getServerName().getAddress().toString(); 228 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); 229 // wait 5 seconds at most 230 conf.setInt(RegionMover.SERVERSTART_WAIT_MAX_KEY, 5); 231 String filename = 232 new Path(TEST_UTIL.getDataTestDir(), "testTargetServerDeadWhenLoading").toString(); 233 // unload the region server 234 try (RegionMover rm = 235 new RegionMoverBuilder(rsName, conf).filename(filename).ack(true).build()) { 236 LOG.info("Unloading " + rs.getServerName()); 237 rm.unload(); 238 assertEquals(0, rs.getNumberOfOnlineRegions()); 239 } 240 String inexistRsName = "whatever:123"; 241 try (RegionMover rm = 242 new RegionMoverBuilder(inexistRsName, conf).filename(filename).ack(true).build()) { 243 // load the regions to an inexist region server, which should fail and return false 244 LOG.info("Loading to an inexist region server {}", inexistRsName); 245 assertFalse(rm.load()); 246 } 247 } 248 249 @Test 250 public void testDecomServerExclusionWithAck() throws Exception { 251 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 252 HRegionServer excludeServer = cluster.getRegionServer(1); 253 List<HRegion> regions = excludeServer.getRegions(); 254 int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions(); 255 TEST_UTIL.getAdmin().decommissionRegionServers( 256 Collections.singletonList(excludeServer.getServerName()), false); 257 258 waitForServerDecom(excludeServer); 259 260 HRegionServer regionServer = cluster.getRegionServer(0); 261 String rsName = regionServer.getServerName().getHostname(); 262 int port = regionServer.getServerName().getPort(); 263 String hostname = rsName + ":" + Integer.toString(port); 264 RegionMoverBuilder rmBuilder = 265 new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration()) 266 .ack(true); 267 268 int targetServerRegions = cluster.getRegionServer(2).getRegions().size(); 269 int sourceServerRegions = regionServer.getRegions().size(); 270 271 try (RegionMover regionMover = rmBuilder.build()) { 272 Assert.assertTrue(regionMover.unload()); 273 LOG.info("Unloading {}", hostname); 274 assertEquals(0, regionServer.getNumberOfOnlineRegions()); 275 assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions()); 276 LOG.info("Before:" + regionsExcludeServer + " After:" + 277 cluster.getRegionServer(1).getNumberOfOnlineRegions()); 278 List<HRegion> regionList = cluster.getRegionServer(1).getRegions(); 279 int index = 0; 280 for (HRegion hRegion : regionList) { 281 Assert.assertEquals(hRegion, regions.get(index++)); 282 } 283 Assert.assertEquals(targetServerRegions + sourceServerRegions, 284 cluster.getRegionServer(2).getNumberOfOnlineRegions()); 285 Assert.assertTrue(regionMover.load()); 286 } 287 288 TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(), 289 Collections.emptyList()); 290 } 291 292 private void waitForServerDecom(HRegionServer excludeServer) { 293 294 TEST_UTIL.waitFor(3000, () -> { 295 try { 296 List<ServerName> decomServers = TEST_UTIL.getAdmin().listDecommissionedRegionServers(); 297 return decomServers.size() == 1 298 && decomServers.get(0).equals(excludeServer.getServerName()); 299 } catch (IOException e) { 300 throw new RuntimeException(e); 301 } 302 }); 303 } 304 305 @Test 306 public void testDecomServerExclusion() throws Exception { 307 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 308 HRegionServer excludeServer = cluster.getRegionServer(0); 309 List<HRegion> regions = excludeServer.getRegions(); 310 int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions(); 311 TEST_UTIL.getAdmin().decommissionRegionServers( 312 Collections.singletonList(excludeServer.getServerName()), false); 313 314 waitForServerDecom(excludeServer); 315 316 HRegionServer sourceRegionServer = cluster.getRegionServer(1); 317 String rsName = sourceRegionServer.getServerName().getHostname(); 318 int port = sourceRegionServer.getServerName().getPort(); 319 String hostname = rsName + ":" + Integer.toString(port); 320 RegionMoverBuilder rmBuilder = 321 new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration()).ack(false); 322 323 int targetServerRegions = cluster.getRegionServer(2).getRegions().size(); 324 int sourceServerRegions = sourceRegionServer.getRegions().size(); 325 326 try (RegionMover regionMover = rmBuilder.build()) { 327 Assert.assertTrue(regionMover.unload()); 328 LOG.info("Unloading {}", hostname); 329 assertEquals(0, sourceRegionServer.getNumberOfOnlineRegions()); 330 assertEquals(regionsExcludeServer, cluster.getRegionServer(0).getNumberOfOnlineRegions()); 331 LOG.info("Before:" + regionsExcludeServer + " After:" + 332 cluster.getRegionServer(1).getNumberOfOnlineRegions()); 333 List<HRegion> regionList = cluster.getRegionServer(0).getRegions(); 334 int index = 0; 335 for (HRegion hRegion : regionList) { 336 Assert.assertEquals(hRegion, regions.get(index++)); 337 } 338 Assert.assertEquals(targetServerRegions + sourceServerRegions, 339 cluster.getRegionServer(2).getNumberOfOnlineRegions()); 340 Assert.assertTrue(regionMover.load()); 341 } 342 343 TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(), 344 Collections.emptyList()); 345 } 346 347 @Test 348 public void testExcludeAndDecomServers() throws Exception { 349 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 350 File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file"); 351 FileWriter fos = new FileWriter(excludeFile); 352 HRegionServer excludeServer = cluster.getRegionServer(1); 353 String excludeHostname = excludeServer.getServerName().getHostname(); 354 int excludeServerPort = excludeServer.getServerName().getPort(); 355 String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort); 356 fos.write(excludeServerName); 357 fos.close(); 358 359 HRegionServer decomServer = cluster.getRegionServer(2); 360 TEST_UTIL.getAdmin().decommissionRegionServers( 361 Collections.singletonList(decomServer.getServerName()), false); 362 363 waitForServerDecom(decomServer); 364 365 HRegionServer regionServer = cluster.getRegionServer(0); 366 String rsName = regionServer.getServerName().getHostname(); 367 int port = regionServer.getServerName().getPort(); 368 String sourceServer = rsName + ":" + Integer.toString(port); 369 RegionMoverBuilder rmBuilder = 370 new RegionMoverBuilder(sourceServer, TEST_UTIL.getConfiguration()) 371 .ack(true) 372 .excludeFile(excludeFile.getCanonicalPath()); 373 try (RegionMover regionMover = rmBuilder.build()) { 374 Assert.assertFalse(regionMover.unload()); 375 } 376 377 TEST_UTIL.getAdmin().recommissionRegionServer(decomServer.getServerName(), 378 Collections.emptyList()); 379 } 380 381}