001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.hbase.master; 020 021import edu.umd.cs.findbugs.annotations.Nullable; 022import java.io.UnsupportedEncodingException; 023import java.nio.charset.StandardCharsets; 024import java.util.HashMap; 025import java.util.List; 026import java.util.Map; 027import java.util.Set; 028 029import org.apache.hadoop.conf.Configuration; 030import org.apache.hadoop.hbase.ClusterMetrics; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HBaseTestingUtility; 033import org.apache.hadoop.hbase.RegionMetrics; 034import org.apache.hadoop.hbase.ServerMetrics; 035import org.apache.hadoop.hbase.ServerName; 036import org.apache.hadoop.hbase.Size; 037import org.apache.hadoop.hbase.Stoppable; 038import org.apache.hadoop.hbase.TableName; 039import org.apache.hadoop.hbase.UserMetrics; 040import org.apache.hadoop.hbase.client.RegionInfo; 041import org.apache.hadoop.hbase.client.RegionStatesCount; 042import org.apache.hadoop.hbase.master.assignment.AssignmentManager; 043import org.apache.hadoop.hbase.replication.ReplicationLoadSink; 044import org.apache.hadoop.hbase.replication.ReplicationLoadSource; 045import org.apache.hadoop.hbase.testclassification.MasterTests; 046import org.apache.hadoop.hbase.testclassification.SmallTests; 047import org.apache.hadoop.hbase.util.Bytes; 048import org.junit.After; 049import org.junit.Before; 050import org.junit.ClassRule; 051import org.junit.Test; 052import org.junit.experimental.categories.Category; 053import org.mockito.Mockito; 054import org.slf4j.Logger; 055import org.slf4j.LoggerFactory; 056 057/** 058 * Test for RegionsRecoveryChore 059 */ 060@Category({MasterTests.class, SmallTests.class}) 061public class TestRegionsRecoveryChore { 062 063 @ClassRule 064 public static final HBaseClassTestRule CLASS_RULE = 065 HBaseClassTestRule.forClass(TestRegionsRecoveryChore.class); 066 067 private static final Logger LOG = LoggerFactory.getLogger(TestRegionsRecoveryChore.class); 068 069 private static final HBaseTestingUtility HBASE_TESTING_UTILITY = new HBaseTestingUtility(); 070 071 private static final String UTF_8_CHARSET = StandardCharsets.UTF_8.name(); 072 073 private HMaster hMaster; 074 075 private AssignmentManager assignmentManager; 076 077 private RegionsRecoveryChore regionsRecoveryChore; 078 079 private static int regionNo; 080 public static final byte[][] REGION_NAME_LIST = new byte[][]{ 081 new byte[]{114, 101, 103, 105, 111, 110, 50, 49, 95, 51}, 082 new byte[]{114, 101, 103, 105, 111, 110, 50, 53, 95, 51}, 083 new byte[]{114, 101, 103, 105, 111, 110, 50, 54, 95, 52}, 084 new byte[]{114, 101, 103, 105, 111, 110, 51, 50, 95, 53}, 085 new byte[]{114, 101, 103, 105, 111, 110, 51, 49, 95, 52}, 086 new byte[]{114, 101, 103, 105, 111, 110, 51, 48, 95, 51}, 087 new byte[]{114, 101, 103, 105, 111, 110, 50, 48, 95, 50}, 088 new byte[]{114, 101, 103, 105, 111, 110, 50, 52, 95, 50}, 089 new byte[]{114, 101, 103, 105, 111, 110, 50, 57, 95, 50}, 090 new byte[]{114, 101, 103, 105, 111, 110, 51, 53, 95, 50}, 091 new byte[]{114, 101, 103, 105, 111, 110, 49, 48, 56, 95, 49, 49} 092 }; 093 094 private Configuration getCustomConf() { 095 Configuration conf = HBASE_TESTING_UTILITY.getConfiguration(); 096 conf.setInt("hbase.master.regions.recovery.check.interval", 100); 097 return conf; 098 } 099 100 @Before 101 public void setUp() throws Exception { 102 this.hMaster = Mockito.mock(HMaster.class); 103 this.assignmentManager = Mockito.mock(AssignmentManager.class); 104 } 105 106 @After 107 public void tearDown() throws Exception { 108 Mockito.verifyNoMoreInteractions(this.hMaster); 109 Mockito.verifyNoMoreInteractions(this.assignmentManager); 110 } 111 112 @Test 113 public void testRegionReopensWithStoreRefConfig() throws Exception { 114 regionNo = 0; 115 ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(4); 116 final Map<ServerName, ServerMetrics> serverMetricsMap = 117 clusterMetrics.getLiveServerMetrics(); 118 LOG.debug("All Region Names with refCount...."); 119 for (ServerMetrics serverMetrics : serverMetricsMap.values()) { 120 Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics(); 121 for (RegionMetrics regionMetrics : regionMetricsMap.values()) { 122 LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " + 123 regionMetrics.getStoreRefCount()); 124 } 125 } 126 Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics); 127 Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager); 128 for (byte[] regionName : REGION_NAME_LIST) { 129 Mockito.when(assignmentManager.getRegionInfo(regionName)) 130 .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName)); 131 } 132 Stoppable stoppable = new StoppableImplementation(); 133 Configuration configuration = getCustomConf(); 134 configuration.setInt("hbase.regions.recovery.store.file.ref.count", 300); 135 regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster); 136 regionsRecoveryChore.chore(); 137 138 // Verify that we need to reopen regions of 2 tables 139 Mockito.verify(hMaster, Mockito.times(2)).reopenRegions(Mockito.any(), Mockito.anyList(), 140 Mockito.anyLong(), Mockito.anyLong()); 141 Mockito.verify(hMaster, Mockito.times(1)).getClusterMetrics(); 142 143 // Verify that we need to reopen total 3 regions that have refCount > 300 144 Mockito.verify(hMaster, Mockito.times(3)).getAssignmentManager(); 145 Mockito.verify(assignmentManager, Mockito.times(3)) 146 .getRegionInfo(Mockito.any()); 147 } 148 149 @Test 150 public void testRegionReopensWithLessThreshold() throws Exception { 151 regionNo = 0; 152 ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(4); 153 final Map<ServerName, ServerMetrics> serverMetricsMap = 154 clusterMetrics.getLiveServerMetrics(); 155 LOG.debug("All Region Names with refCount...."); 156 for (ServerMetrics serverMetrics : serverMetricsMap.values()) { 157 Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics(); 158 for (RegionMetrics regionMetrics : regionMetricsMap.values()) { 159 LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " + 160 regionMetrics.getStoreRefCount()); 161 } 162 } 163 Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics); 164 Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager); 165 for (byte[] regionName : REGION_NAME_LIST) { 166 Mockito.when(assignmentManager.getRegionInfo(regionName)) 167 .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName)); 168 } 169 Stoppable stoppable = new StoppableImplementation(); 170 Configuration configuration = getCustomConf(); 171 configuration.setInt("hbase.regions.recovery.store.file.ref.count", 400); 172 regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster); 173 regionsRecoveryChore.chore(); 174 175 // Verify that we need to reopen regions of only 1 table 176 Mockito.verify(hMaster, Mockito.times(1)).reopenRegions(Mockito.any(), Mockito.anyList(), 177 Mockito.anyLong(), Mockito.anyLong()); 178 Mockito.verify(hMaster, Mockito.times(1)).getClusterMetrics(); 179 180 // Verify that we need to reopen only 1 region with refCount > 400 181 Mockito.verify(hMaster, Mockito.times(1)).getAssignmentManager(); 182 Mockito.verify(assignmentManager, Mockito.times(1)) 183 .getRegionInfo(Mockito.any()); 184 } 185 186 @Test 187 public void testRegionReopensWithoutStoreRefConfig() throws Exception { 188 regionNo = 0; 189 ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(10); 190 final Map<ServerName, ServerMetrics> serverMetricsMap = 191 clusterMetrics.getLiveServerMetrics(); 192 LOG.debug("All Region Names with refCount...."); 193 for (ServerMetrics serverMetrics : serverMetricsMap.values()) { 194 Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics(); 195 for (RegionMetrics regionMetrics : regionMetricsMap.values()) { 196 LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " + 197 regionMetrics.getStoreRefCount()); 198 } 199 } 200 Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics); 201 Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager); 202 for (byte[] regionName : REGION_NAME_LIST) { 203 Mockito.when(assignmentManager.getRegionInfo(regionName)) 204 .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName)); 205 } 206 Stoppable stoppable = new StoppableImplementation(); 207 Configuration configuration = getCustomConf(); 208 configuration.unset("hbase.regions.recovery.store.file.ref.count"); 209 regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster); 210 regionsRecoveryChore.chore(); 211 212 // Verify that by default the feature is turned off so no regions 213 // should be reopened 214 Mockito.verify(hMaster, Mockito.times(0)).reopenRegions(Mockito.any(), Mockito.anyList(), 215 Mockito.anyLong(), Mockito.anyLong()); 216 217 // default maxCompactedStoreFileRefCount is -1 (no regions to be reopened using AM) 218 Mockito.verify(hMaster, Mockito.times(0)).getAssignmentManager(); 219 Mockito.verify(assignmentManager, Mockito.times(0)) 220 .getRegionInfo(Mockito.any()); 221 } 222 223 private static ClusterMetrics getClusterMetrics(int noOfLiveServer) { 224 ClusterMetrics clusterMetrics = new ClusterMetrics() { 225 226 @Nullable 227 @Override 228 public String getHBaseVersion() { 229 return null; 230 } 231 232 @Override 233 public List<ServerName> getDeadServerNames() { 234 return null; 235 } 236 237 @Override 238 public Map<ServerName, ServerMetrics> getLiveServerMetrics() { 239 Map<ServerName, ServerMetrics> liveServerMetrics = new HashMap<>(); 240 for (int i = 0; i < noOfLiveServer; i++) { 241 ServerName serverName = ServerName.valueOf("rs_" + i, 16010, 12345); 242 liveServerMetrics.put(serverName, TestRegionsRecoveryChore.getServerMetrics(i + 3)); 243 } 244 return liveServerMetrics; 245 } 246 247 @Nullable 248 @Override 249 public ServerName getMasterName() { 250 return null; 251 } 252 253 @Override 254 public List<ServerName> getBackupMasterNames() { 255 return null; 256 } 257 258 @Override 259 public List<RegionState> getRegionStatesInTransition() { 260 return null; 261 } 262 263 @Nullable 264 @Override 265 public String getClusterId() { 266 return null; 267 } 268 269 @Override 270 public List<String> getMasterCoprocessorNames() { 271 return null; 272 } 273 274 @Nullable 275 @Override 276 public Boolean getBalancerOn() { 277 return null; 278 } 279 280 @Override 281 public int getMasterInfoPort() { 282 return 0; 283 } 284 285 @Override 286 public List<ServerName> getServersName() { 287 return null; 288 } 289 290 @Override 291 public Map<TableName, RegionStatesCount> getTableRegionStatesCount() { 292 return null; 293 } 294 295 }; 296 return clusterMetrics; 297 } 298 299 private static ServerMetrics getServerMetrics(int noOfRegions) { 300 ServerMetrics serverMetrics = new ServerMetrics() { 301 302 @Override 303 public ServerName getServerName() { 304 return null; 305 } 306 307 @Override 308 public long getRequestCountPerSecond() { 309 return 0; 310 } 311 312 @Override 313 public long getRequestCount() { 314 return 0; 315 } 316 317 @Override 318 public Size getUsedHeapSize() { 319 return null; 320 } 321 322 @Override 323 public Size getMaxHeapSize() { 324 return null; 325 } 326 327 @Override 328 public int getInfoServerPort() { 329 return 0; 330 } 331 332 @Override 333 public List<ReplicationLoadSource> getReplicationLoadSourceList() { 334 return null; 335 } 336 337 @Override 338 public Map<String, List<ReplicationLoadSource>> getReplicationLoadSourceMap() { 339 return null; 340 } 341 342 @Nullable 343 @Override 344 public ReplicationLoadSink getReplicationLoadSink() { 345 return null; 346 } 347 348 @Override 349 public Map<byte[], RegionMetrics> getRegionMetrics() { 350 Map<byte[], RegionMetrics> regionMetricsMap = new HashMap<>(); 351 for (int i = 0; i < noOfRegions; i++) { 352 byte[] regionName = Bytes.toBytes("region" + regionNo + "_" + i); 353 regionMetricsMap.put(regionName, 354 TestRegionsRecoveryChore.getRegionMetrics(regionName, 100 * i)); 355 ++regionNo; 356 } 357 return regionMetricsMap; 358 } 359 360 @Override public Map<byte[], UserMetrics> getUserMetrics() { 361 return new HashMap<>(); 362 } 363 364 @Override 365 public Set<String> getCoprocessorNames() { 366 return null; 367 } 368 369 @Override 370 public long getReportTimestamp() { 371 return 0; 372 } 373 374 @Override 375 public long getLastReportTimestamp() { 376 return 0; 377 } 378 379 }; 380 return serverMetrics; 381 } 382 383 private static RegionMetrics getRegionMetrics(byte[] regionName, int compactedStoreRefCount) { 384 RegionMetrics regionMetrics = new RegionMetrics() { 385 386 @Override 387 public byte[] getRegionName() { 388 return regionName; 389 } 390 391 @Override 392 public int getStoreCount() { 393 return 0; 394 } 395 396 @Override 397 public int getStoreFileCount() { 398 return 0; 399 } 400 401 @Override 402 public Size getStoreFileSize() { 403 return null; 404 } 405 406 @Override 407 public Size getMemStoreSize() { 408 return null; 409 } 410 411 @Override 412 public long getReadRequestCount() { 413 return 0; 414 } 415 416 @Override 417 public long getWriteRequestCount() { 418 return 0; 419 } 420 421 @Override 422 public long getFilteredReadRequestCount() { 423 return 0; 424 } 425 426 @Override 427 public Size getStoreFileIndexSize() { 428 return null; 429 } 430 431 @Override 432 public Size getStoreFileRootLevelIndexSize() { 433 return null; 434 } 435 436 @Override 437 public Size getStoreFileUncompressedDataIndexSize() { 438 return null; 439 } 440 441 @Override 442 public Size getBloomFilterSize() { 443 return null; 444 } 445 446 @Override 447 public long getCompactingCellCount() { 448 return 0; 449 } 450 451 @Override 452 public long getCompactedCellCount() { 453 return 0; 454 } 455 456 @Override 457 public long getCompletedSequenceId() { 458 return 0; 459 } 460 461 @Override 462 public Map<byte[], Long> getStoreSequenceId() { 463 return null; 464 } 465 466 @Override 467 public Size getUncompressedStoreFileSize() { 468 return null; 469 } 470 471 @Override 472 public float getDataLocality() { 473 return 0; 474 } 475 476 @Override 477 public long getLastMajorCompactionTimestamp() { 478 return 0; 479 } 480 481 @Override 482 public int getStoreRefCount() { 483 return compactedStoreRefCount; 484 } 485 486 @Override 487 public int getMaxCompactedStoreFileRefCount() { 488 return compactedStoreRefCount; 489 } 490 491 }; 492 return regionMetrics; 493 } 494 495 private static RegionInfo getRegionInfo(byte[] regionNameBytes) { 496 RegionInfo regionInfo = new RegionInfo() { 497 498 @Override 499 public String getShortNameToLog() { 500 return null; 501 } 502 503 @Override 504 public long getRegionId() { 505 return 0; 506 } 507 508 @Override 509 public byte[] getRegionName() { 510 return new byte[0]; 511 } 512 513 @Override 514 public String getRegionNameAsString() { 515 try { 516 return new String(regionNameBytes, UTF_8_CHARSET); 517 } catch (UnsupportedEncodingException e) { 518 return ""; 519 } 520 } 521 522 @Override 523 public String getEncodedName() { 524 return null; 525 } 526 527 @Override 528 public byte[] getEncodedNameAsBytes() { 529 return new byte[0]; 530 } 531 532 @Override 533 public byte[] getStartKey() { 534 return new byte[0]; 535 } 536 537 @Override 538 public byte[] getEndKey() { 539 return new byte[0]; 540 } 541 542 @Override 543 public TableName getTable() { 544 String regionName; 545 try { 546 regionName = new String(regionNameBytes, UTF_8_CHARSET); 547 } catch (UnsupportedEncodingException e) { 548 regionName = ""; 549 } 550 int regionNo = Integer.parseInt(regionName.split("_")[1]); 551 TableName tableName = TableName.valueOf("table_" + regionNo % 3); 552 return tableName; 553 } 554 555 @Override 556 public int getReplicaId() { 557 return 0; 558 } 559 560 @Override 561 public boolean isSplit() { 562 return false; 563 } 564 565 @Override 566 public boolean isOffline() { 567 return false; 568 } 569 570 @Override 571 public boolean isSplitParent() { 572 return false; 573 } 574 575 @Override 576 public boolean isMetaRegion() { 577 return false; 578 } 579 580 @Override 581 public boolean containsRange(byte[] rangeStartKey, byte[] rangeEndKey) { 582 return false; 583 } 584 585 @Override 586 public boolean containsRow(byte[] row) { 587 return false; 588 } 589 590 }; 591 return regionInfo; 592 } 593 594 /** 595 * Simple helper class that just keeps track of whether or not its stopped. 596 */ 597 private static class StoppableImplementation implements Stoppable { 598 599 private volatile boolean stop = false; 600 601 @Override 602 public void stop(String why) { 603 this.stop = true; 604 } 605 606 @Override 607 public boolean isStopped() { 608 return this.stop; 609 } 610 611 } 612 613}