001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.normalizer; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertNotNull; 023import static org.junit.Assert.assertTrue; 024import java.io.IOException; 025import java.util.Comparator; 026import java.util.List; 027import java.util.concurrent.TimeUnit; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseTestingUtility; 030import org.apache.hadoop.hbase.HConstants; 031import org.apache.hadoop.hbase.MetaTableAccessor; 032import org.apache.hadoop.hbase.NamespaceDescriptor; 033import org.apache.hadoop.hbase.RegionMetrics; 034import org.apache.hadoop.hbase.ServerName; 035import org.apache.hadoop.hbase.Size; 036import org.apache.hadoop.hbase.TableName; 037import org.apache.hadoop.hbase.Waiter.ExplainingPredicate; 038import org.apache.hadoop.hbase.client.Admin; 039import org.apache.hadoop.hbase.client.Put; 040import org.apache.hadoop.hbase.client.RegionInfo; 041import org.apache.hadoop.hbase.client.Table; 042import org.apache.hadoop.hbase.client.TableDescriptor; 043import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 044import org.apache.hadoop.hbase.master.HMaster; 045import org.apache.hadoop.hbase.master.MasterServices; 046import org.apache.hadoop.hbase.master.TableNamespaceManager; 047import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; 048import org.apache.hadoop.hbase.namespace.TestNamespaceAuditor; 049import org.apache.hadoop.hbase.quotas.QuotaUtil; 050import org.apache.hadoop.hbase.regionserver.HRegion; 051import org.apache.hadoop.hbase.regionserver.Region; 052import org.apache.hadoop.hbase.testclassification.MasterTests; 053import org.apache.hadoop.hbase.testclassification.MediumTests; 054import org.apache.hadoop.hbase.util.Bytes; 055import org.apache.hadoop.hbase.util.LoadTestKVGenerator; 056import org.junit.AfterClass; 057import org.junit.Before; 058import org.junit.BeforeClass; 059import org.junit.ClassRule; 060import org.junit.Rule; 061import org.junit.Test; 062import org.junit.experimental.categories.Category; 063import org.junit.rules.TestName; 064import org.slf4j.Logger; 065import org.slf4j.LoggerFactory; 066 067/** 068 * Testing {@link SimpleRegionNormalizer} on minicluster. 069 */ 070@Category({MasterTests.class, MediumTests.class}) 071public class TestSimpleRegionNormalizerOnCluster { 072 private static final Logger LOG = 073 LoggerFactory.getLogger(TestSimpleRegionNormalizerOnCluster.class); 074 075 @ClassRule 076 public static final HBaseClassTestRule CLASS_RULE = 077 HBaseClassTestRule.forClass(TestSimpleRegionNormalizerOnCluster.class); 078 079 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 080 private static final byte[] FAMILY_NAME = Bytes.toBytes("fam"); 081 082 private static Admin admin; 083 private static HMaster master; 084 085 @Rule 086 public TestName name = new TestName(); 087 088 @BeforeClass 089 public static void beforeAllTests() throws Exception { 090 // we will retry operations when PleaseHoldException is thrown 091 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3); 092 TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); 093 094 // no way for the test to set the regionId on a created region, so disable this feature. 095 TEST_UTIL.getConfiguration().setInt("hbase.normalizer.merge.min_region_age.days", 0); 096 097 TEST_UTIL.startMiniCluster(1); 098 TestNamespaceAuditor.waitForQuotaInitialize(TEST_UTIL); 099 admin = TEST_UTIL.getAdmin(); 100 master = TEST_UTIL.getHBaseCluster().getMaster(); 101 assertNotNull(master); 102 } 103 104 @AfterClass 105 public static void afterAllTests() throws Exception { 106 TEST_UTIL.shutdownMiniCluster(); 107 } 108 109 @Before 110 public void before() throws IOException { 111 // disable the normalizer ahead of time, let the test enable it when its ready. 112 admin.normalizerSwitch(false); 113 } 114 115 @Test 116 public void testHonorsNormalizerSwitch() throws IOException { 117 assertFalse(admin.isNormalizerEnabled()); 118 assertFalse(admin.normalize()); 119 assertFalse(admin.normalizerSwitch(true)); 120 assertTrue(admin.normalize()); 121 } 122 123 /** 124 * Test that disabling normalizer via table configuration is honored. There's 125 * no side-effect to look for (other than a log message), so normalize two 126 * tables, one with the disabled setting, and look for change in one and no 127 * change in the other. 128 */ 129 @Test 130 public void testHonorsNormalizerTableSetting() throws Exception { 131 final TableName tn1 = TableName.valueOf(name.getMethodName() + "1"); 132 final TableName tn2 = TableName.valueOf(name.getMethodName() + "2"); 133 final TableName tn3 = TableName.valueOf(name.getMethodName() + "3"); 134 135 try { 136 final int tn1RegionCount = createTableBegsSplit(tn1, true, false); 137 final int tn2RegionCount = createTableBegsSplit(tn2, false, false); 138 final int tn3RegionCount = createTableBegsSplit(tn3, true, true); 139 140 assertFalse(admin.normalizerSwitch(true)); 141 assertTrue(admin.normalize()); 142 waitForTableSplit(tn1, tn1RegionCount + 1); 143 144 // confirm that tn1 has (tn1RegionCount + 1) number of regions. 145 // tn2 has tn2RegionCount number of regions because normalizer has not been enabled on it. 146 // tn3 has tn3RegionCount number of regions because two plans are run: 147 // 1. split one region to two 148 // 2. merge two regions into one 149 // and hence, total number of regions for tn3 remains same 150 assertEquals( 151 tn1 + " should have split.", 152 tn1RegionCount + 1, 153 MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tn1)); 154 assertEquals( 155 tn2 + " should not have split.", 156 tn2RegionCount, 157 MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tn2)); 158 waitForTableRegionCount(tn3, tn3RegionCount); 159 } finally { 160 dropIfExists(tn1); 161 dropIfExists(tn2); 162 dropIfExists(tn3); 163 } 164 } 165 166 @Test 167 public void testRegionNormalizationSplitWithoutQuotaLimit() throws Exception { 168 testRegionNormalizationSplit(false); 169 } 170 171 @Test 172 public void testRegionNormalizationSplitWithQuotaLimit() throws Exception { 173 testRegionNormalizationSplit(true); 174 } 175 176 void testRegionNormalizationSplit(boolean limitedByQuota) throws Exception { 177 TableName tableName = null; 178 try { 179 tableName = limitedByQuota 180 ? buildTableNameForQuotaTest(name.getMethodName()) 181 : TableName.valueOf(name.getMethodName()); 182 183 final int currentRegionCount = createTableBegsSplit(tableName, true, false); 184 final long existingSkippedSplitCount = master.getRegionNormalizer() 185 .getSkippedCount(PlanType.SPLIT); 186 assertFalse(admin.normalizerSwitch(true)); 187 assertTrue(admin.normalize()); 188 if (limitedByQuota) { 189 waitForSkippedSplits(master, existingSkippedSplitCount); 190 assertEquals( 191 tableName + " should not have split.", 192 currentRegionCount, 193 MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName)); 194 } else { 195 waitForTableSplit(tableName, currentRegionCount + 1); 196 assertEquals( 197 tableName + " should have split.", 198 currentRegionCount + 1, 199 MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName)); 200 } 201 } finally { 202 dropIfExists(tableName); 203 } 204 } 205 206 @Test 207 public void testRegionNormalizationMerge() throws Exception { 208 final TableName tableName = TableName.valueOf(name.getMethodName()); 209 try { 210 final int currentRegionCount = createTableBegsMerge(tableName); 211 assertFalse(admin.normalizerSwitch(true)); 212 assertTrue(admin.normalize()); 213 waitForTableMerge(tableName, currentRegionCount - 1); 214 assertEquals( 215 tableName + " should have merged.", 216 currentRegionCount - 1, 217 MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName)); 218 } finally { 219 dropIfExists(tableName); 220 } 221 } 222 223 private static TableName buildTableNameForQuotaTest(final String methodName) throws IOException { 224 String nsp = "np2"; 225 NamespaceDescriptor nspDesc = 226 NamespaceDescriptor.create(nsp) 227 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5") 228 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build(); 229 admin.createNamespace(nspDesc); 230 return TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + methodName); 231 } 232 233 private static void waitForSkippedSplits(final HMaster master, 234 final long existingSkippedSplitCount) throws Exception { 235 TEST_UTIL.waitFor(TimeUnit.MINUTES.toMillis(5), new ExplainingPredicate<Exception>() { 236 @Override public String explainFailure() { 237 return "waiting to observe split attempt and skipped."; 238 } 239 @Override public boolean evaluate() { 240 final long skippedSplitCount = master.getRegionNormalizer().getSkippedCount(PlanType.SPLIT); 241 return skippedSplitCount > existingSkippedSplitCount; 242 } 243 }); 244 } 245 246 private static void waitForTableRegionCount(final TableName tableName, 247 final int targetRegionCount) throws IOException { 248 TEST_UTIL.waitFor(TimeUnit.MINUTES.toMillis(5), new ExplainingPredicate<IOException>() { 249 @Override 250 public String explainFailure() { 251 return "expected " + targetRegionCount + " number of regions for table " + tableName; 252 } 253 @Override 254 public boolean evaluate() throws IOException { 255 final int currentRegionCount = 256 MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName); 257 return currentRegionCount == targetRegionCount; 258 } 259 }); 260 } 261 262 private static void waitForTableSplit(final TableName tableName, final int targetRegionCount) 263 throws IOException { 264 TEST_UTIL.waitFor(TimeUnit.MINUTES.toMillis(5), new ExplainingPredicate<IOException>() { 265 @Override public String explainFailure() { 266 return "expected normalizer to split region."; 267 } 268 @Override public boolean evaluate() throws IOException { 269 final int currentRegionCount = 270 MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName); 271 return currentRegionCount >= targetRegionCount; 272 } 273 }); 274 } 275 276 private static void waitForTableMerge(final TableName tableName, final int targetRegionCount) 277 throws IOException { 278 TEST_UTIL.waitFor(TimeUnit.MINUTES.toMillis(5), new ExplainingPredicate<IOException>() { 279 @Override public String explainFailure() { 280 return "expected normalizer to merge regions."; 281 } 282 @Override public boolean evaluate() throws IOException { 283 final int currentRegionCount = 284 MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName); 285 return currentRegionCount <= targetRegionCount; 286 } 287 }); 288 } 289 290 private static List<HRegion> generateTestData(final TableName tableName, 291 final int... regionSizesMb) throws IOException { 292 final List<HRegion> generatedRegions; 293 final int numRegions = regionSizesMb.length; 294 try (Table ignored = TEST_UTIL.createMultiRegionTable(tableName, FAMILY_NAME, numRegions)) { 295 // Need to get sorted list of regions here 296 generatedRegions = TEST_UTIL.getHBaseCluster().getRegions(tableName); 297 generatedRegions.sort(Comparator.comparing(HRegion::getRegionInfo, RegionInfo.COMPARATOR)); 298 assertEquals(numRegions, generatedRegions.size()); 299 for (int i = 0; i < numRegions; i++) { 300 HRegion region = generatedRegions.get(i); 301 generateTestData(region, regionSizesMb[i]); 302 region.flush(true); 303 } 304 } 305 return generatedRegions; 306 } 307 308 private static void generateTestData(Region region, int numRows) throws IOException { 309 // generating 1Mb values 310 LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(1024 * 1024, 1024 * 1024); 311 for (int i = 0; i < numRows; ++i) { 312 byte[] key = Bytes.add(region.getRegionInfo().getStartKey(), Bytes.toBytes(i)); 313 for (int j = 0; j < 1; ++j) { 314 Put put = new Put(key); 315 byte[] col = Bytes.toBytes(String.valueOf(j)); 316 byte[] value = dataGenerator.generateRandomSizeValue(key, col); 317 put.addColumn(FAMILY_NAME, col, value); 318 region.put(put); 319 } 320 } 321 } 322 323 private static double getRegionSizeMB(final MasterServices masterServices, 324 final RegionInfo regionInfo) { 325 final ServerName sn = masterServices.getAssignmentManager() 326 .getRegionStates() 327 .getRegionServerOfRegion(regionInfo); 328 final RegionMetrics regionLoad = masterServices.getServerManager() 329 .getLoad(sn) 330 .getRegionMetrics() 331 .get(regionInfo.getRegionName()); 332 if (regionLoad == null) { 333 LOG.debug("{} was not found in RegionsLoad", regionInfo.getRegionNameAsString()); 334 return -1; 335 } 336 return regionLoad.getStoreFileSize().get(Size.Unit.MEGABYTE); 337 } 338 339 /** 340 * create a table with 5 regions, having region sizes so as to provoke a split 341 * of the largest region. 342 * <ul> 343 * <li>total table size: 12</li> 344 * <li>average region size: 2.4</li> 345 * <li>split threshold: 2.4 * 2 = 4.8</li> 346 * </ul> 347 */ 348 private static int createTableBegsSplit(final TableName tableName, 349 final boolean normalizerEnabled, final boolean isMergeEnabled) 350 throws IOException { 351 final List<HRegion> generatedRegions = generateTestData(tableName, 1, 1, 2, 3, 5); 352 assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName)); 353 admin.flush(tableName); 354 355 final TableDescriptor td = TableDescriptorBuilder.newBuilder(admin.getDescriptor(tableName)) 356 .setNormalizationEnabled(normalizerEnabled) 357 .setMergeEnabled(isMergeEnabled) 358 .build(); 359 admin.modifyTable(td); 360 361 // make sure relatively accurate region statistics are available for the test table. use 362 // the last/largest region as clue. 363 TEST_UTIL.waitFor(TimeUnit.MINUTES.toMillis(1), new ExplainingPredicate<IOException>() { 364 @Override public String explainFailure() { 365 return "expected largest region to be >= 4mb."; 366 } 367 @Override public boolean evaluate() { 368 return generatedRegions.stream() 369 .mapToDouble(val -> getRegionSizeMB(master, val.getRegionInfo())) 370 .allMatch(val -> val > 0) 371 && getRegionSizeMB(master, generatedRegions.get(4).getRegionInfo()) >= 4.0; 372 } 373 }); 374 return 5; 375 } 376 377 /** 378 * create a table with 5 regions, having region sizes so as to provoke a merge 379 * of the smallest regions. 380 * <ul> 381 * <li>total table size: 13</li> 382 * <li>average region size: 2.6</li> 383 * <li>sum of sizes of first two regions < average</li> 384 * </ul> 385 */ 386 private static int createTableBegsMerge(final TableName tableName) throws IOException { 387 // create 5 regions with sizes to trigger merge of small regions 388 final List<HRegion> generatedRegions = generateTestData(tableName, 1, 1, 3, 3, 5); 389 assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName)); 390 admin.flush(tableName); 391 392 final TableDescriptor td = TableDescriptorBuilder.newBuilder(admin.getDescriptor(tableName)) 393 .setNormalizationEnabled(true) 394 .build(); 395 admin.modifyTable(td); 396 397 // make sure relatively accurate region statistics are available for the test table. use 398 // the last/largest region as clue. 399 LOG.debug("waiting for region statistics to settle."); 400 TEST_UTIL.waitFor(TimeUnit.MINUTES.toMillis(1), new ExplainingPredicate<IOException>() { 401 @Override public String explainFailure() { 402 return "expected largest region to be >= 4mb."; 403 } 404 @Override public boolean evaluate() { 405 return generatedRegions.stream() 406 .mapToDouble(val -> getRegionSizeMB(master, val.getRegionInfo())) 407 .allMatch(val -> val > 0) 408 && getRegionSizeMB(master, generatedRegions.get(4).getRegionInfo()) >= 4.0; 409 } 410 }); 411 return 5; 412 } 413 414 private static void dropIfExists(final TableName tableName) throws IOException { 415 if (tableName != null && admin.tableExists(tableName)) { 416 if (admin.isTableEnabled(tableName)) { 417 admin.disableTable(tableName); 418 } 419 admin.deleteTable(tableName); 420 } 421 } 422}