001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.util; 019 020import java.util.ArrayList; 021import java.util.List; 022import java.util.concurrent.TimeUnit; 023import java.util.stream.Collectors; 024import org.apache.hadoop.hbase.HBaseClassTestRule; 025import org.apache.hadoop.hbase.HBaseTestingUtil; 026import org.apache.hadoop.hbase.HRegionLocation; 027import org.apache.hadoop.hbase.ServerName; 028import org.apache.hadoop.hbase.SingleProcessHBaseCluster; 029import org.apache.hadoop.hbase.TableName; 030import org.apache.hadoop.hbase.client.Admin; 031import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 032import org.apache.hadoop.hbase.client.Put; 033import org.apache.hadoop.hbase.client.RegionInfoBuilder; 034import org.apache.hadoop.hbase.client.Table; 035import org.apache.hadoop.hbase.client.TableDescriptor; 036import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 037import org.apache.hadoop.hbase.master.RegionState; 038import org.apache.hadoop.hbase.regionserver.HRegion; 039import org.apache.hadoop.hbase.regionserver.HRegionServer; 040import org.apache.hadoop.hbase.testclassification.LargeTests; 041import org.apache.hadoop.hbase.testclassification.MiscTests; 042import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; 043import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 044import org.apache.hadoop.hbase.zookeeper.ZNodePaths; 045import org.junit.After; 046import org.junit.AfterClass; 047import org.junit.Assert; 048import org.junit.Before; 049import org.junit.BeforeClass; 050import org.junit.ClassRule; 051import org.junit.Rule; 052import org.junit.Test; 053import org.junit.experimental.categories.Category; 054import org.junit.rules.TestName; 055import org.slf4j.Logger; 056import org.slf4j.LoggerFactory; 057 058/** 059 * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test 060 * exclude functionality useful for rack decommissioning 061 */ 062@Category({ MiscTests.class, LargeTests.class }) 063public class TestRegionMover2 { 064 065 @ClassRule 066 public static final HBaseClassTestRule CLASS_RULE = 067 HBaseClassTestRule.forClass(TestRegionMover2.class); 068 069 @Rule 070 public TestName name = new TestName(); 071 072 private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover2.class); 073 074 private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); 075 076 @BeforeClass 077 public static void setUpBeforeClass() throws Exception { 078 TEST_UTIL.startMiniCluster(3); 079 TEST_UTIL.getAdmin().balancerSwitch(false, true); 080 } 081 082 @AfterClass 083 public static void tearDownAfterClass() throws Exception { 084 TEST_UTIL.shutdownMiniCluster(); 085 } 086 087 @Before 088 public void setUp() throws Exception { 089 final TableName tableName = TableName.valueOf(name.getMethodName()); 090 TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName) 091 .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build(); 092 int startKey = 0; 093 int endKey = 80000; 094 TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9); 095 } 096 097 @After 098 public void tearDown() throws Exception { 099 final TableName tableName = TableName.valueOf(name.getMethodName()); 100 TEST_UTIL.getAdmin().disableTable(tableName); 101 TEST_UTIL.getAdmin().deleteTable(tableName); 102 } 103 104 @Test 105 public void testWithMergedRegions() throws Exception { 106 final TableName tableName = TableName.valueOf(name.getMethodName()); 107 SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 108 Admin admin = TEST_UTIL.getAdmin(); 109 Table table = TEST_UTIL.getConnection().getTable(tableName); 110 List<Put> puts = new ArrayList<>(); 111 for (int i = 0; i < 10000; i++) { 112 puts.add(new Put(Bytes.toBytes("rowkey_" + i)).addColumn(Bytes.toBytes("fam1"), 113 Bytes.toBytes("q1"), Bytes.toBytes("val_" + i))); 114 } 115 table.put(puts); 116 admin.flush(tableName); 117 HRegionServer regionServer = cluster.getRegionServer(0); 118 String rsName = regionServer.getServerName().getAddress().toString(); 119 int numRegions = regionServer.getNumberOfOnlineRegions(); 120 List<HRegion> hRegions = regionServer.getRegions().stream() 121 .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName)) 122 .collect(Collectors.toList()); 123 RegionMover.RegionMoverBuilder rmBuilder = 124 new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true) 125 .maxthreads(8); 126 try (RegionMover rm = rmBuilder.build()) { 127 LOG.debug("Unloading {}", regionServer.getServerName()); 128 rm.unload(); 129 Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions()); 130 LOG.debug("Successfully Unloaded, now Loading"); 131 admin.mergeRegionsAsync(new byte[][] { hRegions.get(0).getRegionInfo().getRegionName(), 132 hRegions.get(1).getRegionInfo().getRegionName() }, true).get(5, TimeUnit.SECONDS); 133 Assert.assertTrue(rm.load()); 134 Assert.assertEquals(numRegions - 2, regionServer.getNumberOfOnlineRegions()); 135 } 136 } 137 138 @Test 139 public void testWithSplitRegions() throws Exception { 140 final TableName tableName = TableName.valueOf(name.getMethodName()); 141 SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 142 Admin admin = TEST_UTIL.getAdmin(); 143 Table table = TEST_UTIL.getConnection().getTable(tableName); 144 List<Put> puts = new ArrayList<>(); 145 for (int i = 10; i < 50000; i++) { 146 puts.add(new Put(Bytes.toBytes(i)).addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("q1"), 147 Bytes.toBytes("val_" + i))); 148 } 149 table.put(puts); 150 admin.flush(tableName); 151 admin.compact(tableName); 152 HRegionServer regionServer = cluster.getRegionServer(0); 153 String rsName = regionServer.getServerName().getAddress().toString(); 154 int numRegions = regionServer.getNumberOfOnlineRegions(); 155 List<HRegion> hRegions = regionServer.getRegions().stream() 156 .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName)) 157 .collect(Collectors.toList()); 158 159 RegionMover.RegionMoverBuilder rmBuilder = 160 new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true) 161 .maxthreads(8); 162 try (RegionMover rm = rmBuilder.build()) { 163 LOG.debug("Unloading {}", regionServer.getServerName()); 164 rm.unload(); 165 Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions()); 166 LOG.debug("Successfully Unloaded, now Loading"); 167 HRegion hRegion = hRegions.get(1); 168 if (hRegion.getRegionInfo().getStartKey().length == 0) { 169 hRegion = hRegions.get(0); 170 } 171 int startKey = 0; 172 int endKey = Integer.MAX_VALUE; 173 if (hRegion.getRegionInfo().getStartKey().length > 0) { 174 startKey = Bytes.toInt(hRegion.getRegionInfo().getStartKey()); 175 } 176 if (hRegion.getRegionInfo().getEndKey().length > 0) { 177 endKey = Bytes.toInt(hRegion.getRegionInfo().getEndKey()); 178 } 179 int midKey = startKey + (endKey - startKey) / 2; 180 admin.splitRegionAsync(hRegion.getRegionInfo().getRegionName(), Bytes.toBytes(midKey)).get(5, 181 TimeUnit.SECONDS); 182 Assert.assertTrue(rm.load()); 183 Assert.assertEquals(numRegions - 1, regionServer.getNumberOfOnlineRegions()); 184 } 185 } 186 187 @Test 188 public void testFailedRegionMove() throws Exception { 189 final TableName tableName = TableName.valueOf(name.getMethodName()); 190 SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 191 Admin admin = TEST_UTIL.getAdmin(); 192 Table table = TEST_UTIL.getConnection().getTable(tableName); 193 List<Put> puts = new ArrayList<>(); 194 for (int i = 0; i < 1000; i++) { 195 puts.add(new Put(Bytes.toBytes("rowkey_" + i)).addColumn(Bytes.toBytes("fam1"), 196 Bytes.toBytes("q1"), Bytes.toBytes("val_" + i))); 197 } 198 table.put(puts); 199 admin.flush(tableName); 200 HRegionServer regionServer = cluster.getRegionServer(0); 201 String rsName = regionServer.getServerName().getAddress().toString(); 202 int numRegions = regionServer.getNumberOfOnlineRegions(); 203 List<HRegion> hRegions = regionServer.getRegions().stream() 204 .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName)) 205 .collect(Collectors.toList()); 206 RegionMover.RegionMoverBuilder rmBuilder = 207 new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true) 208 .maxthreads(8); 209 try (RegionMover rm = rmBuilder.build()) { 210 LOG.debug("Unloading {}", regionServer.getServerName()); 211 rm.unload(); 212 Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions()); 213 LOG.debug("Successfully Unloaded, now Loading"); 214 admin.offline(hRegions.get(0).getRegionInfo().getRegionName()); 215 // loading regions will fail because of offline region 216 Assert.assertFalse(rm.load()); 217 } 218 } 219 220 public void loadDummyDataInTable(TableName tableName) throws Exception { 221 Admin admin = TEST_UTIL.getAdmin(); 222 Table table = TEST_UTIL.getConnection().getTable(tableName); 223 List<Put> puts = new ArrayList<>(); 224 for (int i = 0; i < 1000; i++) { 225 puts.add(new Put(Bytes.toBytes("rowkey_" + i)).addColumn(Bytes.toBytes("fam1"), 226 Bytes.toBytes("q1"), Bytes.toBytes("val_" + i))); 227 } 228 table.put(puts); 229 admin.flush(tableName); 230 } 231 232 @Test 233 public void testIsolateSingleRegionOnTheSameServer() throws Exception { 234 final TableName tableName = TableName.valueOf(name.getMethodName()); 235 loadDummyDataInTable(tableName); 236 ServerName sourceServerName = findSourceServerName(tableName); 237 // Isolating 1 region on the same region server. 238 regionIsolationOperation(sourceServerName, sourceServerName, 1, false); 239 } 240 241 @Test 242 public void testIsolateSingleRegionOnTheDifferentServer() throws Exception { 243 final TableName tableName = TableName.valueOf(name.getMethodName()); 244 loadDummyDataInTable(tableName); 245 ServerName sourceServerName = findSourceServerName(tableName); 246 ServerName destinationServerName = findDestinationServerName(sourceServerName); 247 // Isolating 1 region on the different region server. 248 regionIsolationOperation(sourceServerName, destinationServerName, 1, false); 249 } 250 251 @Test 252 public void testIsolateMultipleRegionsOnTheSameServer() throws Exception { 253 final TableName tableName = TableName.valueOf(name.getMethodName()); 254 loadDummyDataInTable(tableName); 255 ServerName sourceServerName = findSourceServerName(tableName); 256 // Isolating 2 regions on the same region server. 257 regionIsolationOperation(sourceServerName, sourceServerName, 2, false); 258 } 259 260 @Test 261 public void testIsolateMultipleRegionsOnTheDifferentServer() throws Exception { 262 final TableName tableName = TableName.valueOf(name.getMethodName()); 263 loadDummyDataInTable(tableName); 264 // Isolating 2 regions on the different region server. 265 ServerName sourceServerName = findSourceServerName(tableName); 266 ServerName destinationServerName = findDestinationServerName(sourceServerName); 267 regionIsolationOperation(sourceServerName, destinationServerName, 2, false); 268 } 269 270 @Test 271 public void testIsolateMetaOnTheSameSever() throws Exception { 272 ServerName metaServerSource = findMetaRSLocation(); 273 regionIsolationOperation(metaServerSource, metaServerSource, 1, true); 274 } 275 276 @Test 277 public void testIsolateMetaOnTheDifferentServer() throws Exception { 278 ServerName metaServerSource = findMetaRSLocation(); 279 ServerName metaServerDestination = findDestinationServerName(metaServerSource); 280 regionIsolationOperation(metaServerSource, metaServerDestination, 1, true); 281 } 282 283 @Test 284 public void testIsolateMetaAndRandomRegionOnTheMetaServer() throws Exception { 285 final TableName tableName = TableName.valueOf(name.getMethodName()); 286 loadDummyDataInTable(tableName); 287 ServerName metaServerSource = findMetaRSLocation(); 288 ServerName randomSeverRegion = findSourceServerName(tableName); 289 regionIsolationOperation(randomSeverRegion, metaServerSource, 2, true); 290 } 291 292 @Test 293 public void testIsolateMetaAndRandomRegionOnTheRandomServer() throws Exception { 294 final TableName tableName = TableName.valueOf(name.getMethodName()); 295 loadDummyDataInTable(tableName); 296 ServerName randomSeverRegion = findSourceServerName(tableName); 297 regionIsolationOperation(randomSeverRegion, randomSeverRegion, 2, true); 298 } 299 300 public ServerName findMetaRSLocation() throws Exception { 301 ZKWatcher zkWatcher = new ZKWatcher(TEST_UTIL.getConfiguration(), null, null); 302 List<HRegionLocation> result = new ArrayList<>(); 303 for (String znode : zkWatcher.getMetaReplicaNodes()) { 304 String path = ZNodePaths.joinZNode(zkWatcher.getZNodePaths().baseZNode, znode); 305 int replicaId = zkWatcher.getZNodePaths().getMetaReplicaIdFromPath(path); 306 RegionState state = MetaTableLocator.getMetaRegionState(zkWatcher, replicaId); 307 result.add(new HRegionLocation(state.getRegion(), state.getServerName())); 308 } 309 return result.get(0).getServerName(); 310 } 311 312 public ServerName findSourceServerName(TableName tableName) throws Exception { 313 SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 314 int numOfRS = cluster.getNumLiveRegionServers(); 315 ServerName sourceServer = null; 316 for (int i = 0; i < numOfRS; i++) { 317 HRegionServer regionServer = cluster.getRegionServer(i); 318 List<HRegion> hRegions = regionServer.getRegions().stream() 319 .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName)) 320 .collect(Collectors.toList()); 321 if (hRegions.size() >= 2) { 322 sourceServer = regionServer.getServerName(); 323 break; 324 } 325 } 326 if (sourceServer == null) { 327 throw new Exception( 328 "This shouln't happen, No RS found with more than 2 regions of table : " + tableName); 329 } 330 return sourceServer; 331 } 332 333 public ServerName findDestinationServerName(ServerName sourceServerName) throws Exception { 334 SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 335 ServerName destinationServerName = null; 336 int numOfRS = cluster.getNumLiveRegionServers(); 337 for (int i = 0; i < numOfRS; i++) { 338 destinationServerName = cluster.getRegionServer(i).getServerName(); 339 if (!destinationServerName.equals(sourceServerName)) { 340 break; 341 } 342 } 343 if (destinationServerName == null) { 344 throw new Exception("This shouldn't happen, No RS found which is different than source RS"); 345 } 346 return destinationServerName; 347 } 348 349 public void regionIsolationOperation(ServerName sourceServerName, 350 ServerName destinationServerName, int numRegionsToIsolate, boolean isolateMetaAlso) 351 throws Exception { 352 final TableName tableName = TableName.valueOf(name.getMethodName()); 353 SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 354 Admin admin = TEST_UTIL.getAdmin(); 355 HRegionServer sourceRS = cluster.getRegionServer(sourceServerName); 356 List<HRegion> hRegions = sourceRS.getRegions().stream() 357 .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName)) 358 .collect(Collectors.toList()); 359 List<String> listOfRegionIDsToIsolate = new ArrayList<>(); 360 for (int i = 0; i < numRegionsToIsolate; i++) { 361 listOfRegionIDsToIsolate.add(hRegions.get(i).getRegionInfo().getEncodedName()); 362 } 363 364 if (isolateMetaAlso) { 365 listOfRegionIDsToIsolate.remove(0); 366 listOfRegionIDsToIsolate.add(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); 367 } 368 369 HRegionServer destinationRS = cluster.getRegionServer(destinationServerName); 370 String destinationRSName = destinationRS.getServerName().getAddress().toString(); 371 RegionMover.RegionMoverBuilder rmBuilder = 372 new RegionMover.RegionMoverBuilder(destinationRSName, TEST_UTIL.getConfiguration()).ack(true) 373 .maxthreads(8).isolateRegionIdArray(listOfRegionIDsToIsolate); 374 try (RegionMover rm = rmBuilder.build()) { 375 LOG.debug("Unloading {} except regions : {}", destinationRS.getServerName(), 376 listOfRegionIDsToIsolate); 377 rm.isolateRegions(); 378 Assert.assertEquals(numRegionsToIsolate, destinationRS.getNumberOfOnlineRegions()); 379 List<HRegion> onlineRegions = destinationRS.getRegions(); 380 for (int i = 0; i < numRegionsToIsolate; i++) { 381 Assert.assertTrue( 382 listOfRegionIDsToIsolate.contains(onlineRegions.get(i).getRegionInfo().getEncodedName())); 383 } 384 LOG.debug("Successfully Isolated " + listOfRegionIDsToIsolate.size() + " regions : " 385 + listOfRegionIDsToIsolate + " on " + destinationRS.getServerName()); 386 } finally { 387 admin.recommissionRegionServer(destinationRS.getServerName(), null); 388 } 389 } 390}