001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.assignment; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertTrue; 022 023import java.io.IOException; 024import java.util.ArrayList; 025import java.util.List; 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.hbase.HBaseClassTestRule; 028import org.apache.hadoop.hbase.HBaseTestingUtility; 029import org.apache.hadoop.hbase.HConstants; 030import org.apache.hadoop.hbase.MetaTableAccessor; 031import org.apache.hadoop.hbase.TableName; 032import org.apache.hadoop.hbase.client.Admin; 033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 034import org.apache.hadoop.hbase.client.Put; 035import org.apache.hadoop.hbase.client.RegionInfo; 036import org.apache.hadoop.hbase.client.Table; 037import org.apache.hadoop.hbase.client.TableDescriptor; 038import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 039import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; 040import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; 041import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility; 042import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; 043import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; 044import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; 045import org.apache.hadoop.hbase.regionserver.HRegion; 046import org.apache.hadoop.hbase.testclassification.MasterTests; 047import org.apache.hadoop.hbase.testclassification.MediumTests; 048import org.apache.hadoop.hbase.util.Bytes; 049import org.apache.hadoop.hbase.util.Threads; 050import org.junit.After; 051import org.junit.AfterClass; 052import org.junit.Before; 053import org.junit.BeforeClass; 054import org.junit.ClassRule; 055import org.junit.Rule; 056import org.junit.Test; 057import org.junit.experimental.categories.Category; 058import org.junit.rules.TestName; 059import org.slf4j.Logger; 060import org.slf4j.LoggerFactory; 061 062@Category({MasterTests.class, MediumTests.class}) 063public class TestMergeTableRegionsProcedure { 064 065 @ClassRule 066 public static final HBaseClassTestRule CLASS_RULE = 067 HBaseClassTestRule.forClass(TestMergeTableRegionsProcedure.class); 068 069 private static final Logger LOG = LoggerFactory.getLogger(TestMergeTableRegionsProcedure.class); 070 @Rule 071 public final TestName name = new TestName(); 072 073 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); 074 075 private static final int initialRegionCount = 4; 076 private final static byte[] FAMILY = Bytes.toBytes("FAMILY"); 077 private static Admin admin; 078 079 private ProcedureMetrics mergeProcMetrics; 080 private ProcedureMetrics assignProcMetrics; 081 private ProcedureMetrics unassignProcMetrics; 082 private long mergeSubmittedCount = 0; 083 private long mergeFailedCount = 0; 084 private long assignSubmittedCount = 0; 085 private long assignFailedCount = 0; 086 private long unassignSubmittedCount = 0; 087 private long unassignFailedCount = 0; 088 089 private static void setupConf(Configuration conf) { 090 // Reduce the maximum attempts to speed up the test 091 conf.setInt("hbase.assignment.maximum.attempts", 3); 092 conf.setInt("hbase.master.maximum.ping.server.attempts", 3); 093 conf.setInt("hbase.master.ping.server.retry.sleep.interval", 1); 094 conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); 095 } 096 097 @BeforeClass 098 public static void setupCluster() throws Exception { 099 setupConf(UTIL.getConfiguration()); 100 UTIL.startMiniCluster(1); 101 admin = UTIL.getAdmin(); 102 } 103 104 @AfterClass 105 public static void cleanupTest() throws Exception { 106 UTIL.shutdownMiniCluster(); 107 } 108 109 @Before 110 public void setup() throws Exception { 111 resetProcExecutorTestingKillFlag(); 112 MasterProcedureTestingUtility.generateNonceGroup(UTIL.getHBaseCluster().getMaster()); 113 MasterProcedureTestingUtility.generateNonce(UTIL.getHBaseCluster().getMaster()); 114 // Turn off balancer so it doesn't cut in and mess up our placements. 115 admin.balancerSwitch(false, true); 116 // Turn off the meta scanner so it don't remove parent on us. 117 UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false); 118 resetProcExecutorTestingKillFlag(); 119 AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager(); 120 mergeProcMetrics = am.getAssignmentManagerMetrics().getMergeProcMetrics(); 121 assignProcMetrics = am.getAssignmentManagerMetrics().getAssignProcMetrics(); 122 unassignProcMetrics = am.getAssignmentManagerMetrics().getUnassignProcMetrics(); 123 } 124 125 @After 126 public void tearDown() throws Exception { 127 resetProcExecutorTestingKillFlag(); 128 for (TableDescriptor htd: admin.listTableDescriptors()) { 129 LOG.info("Tear down, remove table=" + htd.getTableName()); 130 UTIL.deleteTable(htd.getTableName()); 131 } 132 } 133 134 private void resetProcExecutorTestingKillFlag() { 135 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 136 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); 137 assertTrue("expected executor to be running", procExec.isRunning()); 138 } 139 140 private int loadARowPerRegion(final Table t, List<RegionInfo> ris) 141 throws IOException { 142 List<Put> puts = new ArrayList<>(); 143 for (RegionInfo ri: ris) { 144 Put put = new Put(ri.getStartKey() == null || ri.getStartKey().length == 0? 145 new byte [] {'a'}: ri.getStartKey()); 146 put.addColumn(HConstants.CATALOG_FAMILY, HConstants.CATALOG_FAMILY, 147 HConstants.CATALOG_FAMILY); 148 puts.add(put); 149 } 150 t.put(puts); 151 return puts.size(); 152 } 153 154 155 /** 156 * This tests two region merges 157 */ 158 @Test 159 public void testMergeTwoRegions() throws Exception { 160 final TableName tableName = TableName.valueOf(this.name.getMethodName()); 161 UTIL.createTable(tableName, new byte[][]{HConstants.CATALOG_FAMILY}, 162 new byte[][]{new byte[]{'b'}, new byte[]{'c'}, new byte[]{'d'}, new byte[]{'e'}}); 163 testMerge(tableName, 2); 164 } 165 166 private void testMerge(TableName tableName, int mergeCount) throws IOException { 167 List<RegionInfo> ris = MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName); 168 int originalRegionCount = ris.size(); 169 assertTrue(originalRegionCount > mergeCount); 170 RegionInfo[] regionsToMerge = ris.subList(0, mergeCount).toArray(new RegionInfo [] {}); 171 int countOfRowsLoaded = 0; 172 try (Table table = UTIL.getConnection().getTable(tableName)) { 173 countOfRowsLoaded = loadARowPerRegion(table, ris); 174 } 175 assertEquals(countOfRowsLoaded, UTIL.countRows(tableName)); 176 177 // collect AM metrics before test 178 collectAssignmentManagerMetrics(); 179 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 180 MergeTableRegionsProcedure proc = 181 new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true); 182 long procId = procExec.submitProcedure(proc); 183 ProcedureTestingUtility.waitProcedure(procExec, procId); 184 ProcedureTestingUtility.assertProcNotFailed(procExec, procId); 185 MetaTableAccessor.fullScanMetaAndPrint(UTIL.getConnection()); 186 assertEquals(originalRegionCount - mergeCount + 1, 187 MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName).size()); 188 189 assertEquals(mergeSubmittedCount + 1, mergeProcMetrics.getSubmittedCounter().getCount()); 190 assertEquals(mergeFailedCount, mergeProcMetrics.getFailedCounter().getCount()); 191 assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount()); 192 assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount()); 193 assertEquals(unassignSubmittedCount + mergeCount, 194 unassignProcMetrics.getSubmittedCounter().getCount()); 195 assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount()); 196 197 // Need to get the references cleaned out. Close of region will move them 198 // to archive so disable and reopen just to get rid of references to later 199 // when the catalogjanitor runs, it can do merged region cleanup. 200 admin.disableTable(tableName); 201 admin.enableTable(tableName); 202 203 // Can I purge the merged regions from hbase:meta? Check that all went 204 // well by looking at the merged row up in hbase:meta. It should have no 205 // more mention of the merged regions; they are purged as last step in 206 // the merged regions cleanup. 207 UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(true); 208 UTIL.getHBaseCluster().getMaster().getCatalogJanitor().triggerNow(); 209 byte [] mergedRegion = proc.getMergedRegion().getRegionName(); 210 while (ris != null && ris.get(0) != null && ris.get(1) != null) { 211 ris = MetaTableAccessor.getMergeRegions(UTIL.getConnection(), mergedRegion); 212 LOG.info("{} {}", Bytes.toStringBinary(mergedRegion), ris); 213 Threads.sleep(1000); 214 } 215 assertEquals(countOfRowsLoaded, UTIL.countRows(tableName)); 216 } 217 218 /** 219 * This tests ten region merges in one go. 220 */ 221 @Test 222 public void testMergeTenRegions() throws Exception { 223 final TableName tableName = TableName.valueOf(this.name.getMethodName()); 224 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 225 UTIL.createMultiRegionTable(tableName, HConstants.CATALOG_FAMILY); 226 testMerge(tableName, 10); 227 } 228 229 /** 230 * This tests two concurrent region merges 231 */ 232 @Test 233 public void testMergeRegionsConcurrently() throws Exception { 234 final TableName tableName = TableName.valueOf("testMergeRegionsConcurrently"); 235 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 236 237 List<RegionInfo> tableRegions = createTable(tableName); 238 239 RegionInfo[] regionsToMerge1 = new RegionInfo[2]; 240 RegionInfo[] regionsToMerge2 = new RegionInfo[2]; 241 regionsToMerge1[0] = tableRegions.get(0); 242 regionsToMerge1[1] = tableRegions.get(1); 243 regionsToMerge2[0] = tableRegions.get(2); 244 regionsToMerge2[1] = tableRegions.get(3); 245 246 // collect AM metrics before test 247 collectAssignmentManagerMetrics(); 248 249 long procId1 = procExec.submitProcedure(new MergeTableRegionsProcedure( 250 procExec.getEnvironment(), regionsToMerge1, true)); 251 long procId2 = procExec.submitProcedure(new MergeTableRegionsProcedure( 252 procExec.getEnvironment(), regionsToMerge2, true)); 253 ProcedureTestingUtility.waitProcedure(procExec, procId1); 254 ProcedureTestingUtility.waitProcedure(procExec, procId2); 255 ProcedureTestingUtility.assertProcNotFailed(procExec, procId1); 256 ProcedureTestingUtility.assertProcNotFailed(procExec, procId2); 257 assertRegionCount(tableName, initialRegionCount - 2); 258 259 assertEquals(mergeSubmittedCount + 2, mergeProcMetrics.getSubmittedCounter().getCount()); 260 assertEquals(mergeFailedCount, mergeProcMetrics.getFailedCounter().getCount()); 261 assertEquals(assignSubmittedCount + 2, assignProcMetrics.getSubmittedCounter().getCount()); 262 assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount()); 263 assertEquals(unassignSubmittedCount + 4, unassignProcMetrics.getSubmittedCounter().getCount()); 264 assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount()); 265 } 266 267 @Test 268 public void testRecoveryAndDoubleExecution() throws Exception { 269 final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution"); 270 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 271 272 List<RegionInfo> tableRegions = createTable(tableName); 273 274 ProcedureTestingUtility.waitNoProcedureRunning(procExec); 275 ProcedureTestingUtility.setKillIfHasParent(procExec, false); 276 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 277 278 RegionInfo[] regionsToMerge = new RegionInfo[2]; 279 regionsToMerge[0] = tableRegions.get(0); 280 regionsToMerge[1] = tableRegions.get(1); 281 282 long procId = procExec.submitProcedure( 283 new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true)); 284 285 // Restart the executor and execute the step twice 286 MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); 287 ProcedureTestingUtility.assertProcNotFailed(procExec, procId); 288 289 assertRegionCount(tableName, initialRegionCount - 1); 290 } 291 292 @Test 293 public void testRollbackAndDoubleExecution() throws Exception { 294 final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution"); 295 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 296 297 List<RegionInfo> tableRegions = createTable(tableName); 298 299 ProcedureTestingUtility.waitNoProcedureRunning(procExec); 300 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 301 302 RegionInfo[] regionsToMerge = new RegionInfo[2]; 303 regionsToMerge[0] = tableRegions.get(0); 304 regionsToMerge[1] = tableRegions.get(1); 305 306 long procId = procExec.submitProcedure( 307 new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true)); 308 309 // Failing before MERGE_TABLE_REGIONS_UPDATE_META we should trigger the rollback 310 // NOTE: the 8 (number of MERGE_TABLE_REGIONS_UPDATE_META step) is 311 // hardcoded, so you have to look at this test at least once when you add a new step. 312 int lastStep = 8; 313 MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep, true); 314 assertEquals(initialRegionCount, UTIL.getAdmin().getRegions(tableName).size()); 315 UTIL.waitUntilAllRegionsAssigned(tableName); 316 List<HRegion> regions = UTIL.getMiniHBaseCluster().getRegions(tableName); 317 assertEquals(initialRegionCount, regions.size()); 318 } 319 320 @Test 321 public void testMergeWithoutPONR() throws Exception { 322 final TableName tableName = TableName.valueOf("testMergeWithoutPONR"); 323 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); 324 325 List<RegionInfo> tableRegions = createTable(tableName); 326 327 ProcedureTestingUtility.waitNoProcedureRunning(procExec); 328 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); 329 330 RegionInfo[] regionsToMerge = new RegionInfo[2]; 331 regionsToMerge[0] = tableRegions.get(0); 332 regionsToMerge[1] = tableRegions.get(1); 333 334 long procId = procExec.submitProcedure( 335 new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true)); 336 337 // Execute until step 9 of split procedure 338 // NOTE: step 9 is after step MERGE_TABLE_REGIONS_UPDATE_META 339 MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, 9, false); 340 341 // Unset Toggle Kill and make ProcExec work correctly 342 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); 343 MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec); 344 ProcedureTestingUtility.waitProcedure(procExec, procId); 345 346 assertRegionCount(tableName, initialRegionCount - 1); 347 } 348 349 private List<RegionInfo> createTable(final TableName tableName) throws Exception { 350 TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) 351 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build(); 352 byte[][] splitRows = new byte[initialRegionCount - 1][]; 353 for (int i = 0; i < splitRows.length; ++i) { 354 splitRows[i] = Bytes.toBytes(String.format("%d", i)); 355 } 356 admin.createTable(desc, splitRows); 357 return assertRegionCount(tableName, initialRegionCount); 358 } 359 360 public List<RegionInfo> assertRegionCount(final TableName tableName, final int nregions) 361 throws Exception { 362 UTIL.waitUntilNoRegionsInTransition(); 363 List<RegionInfo> tableRegions = admin.getRegions(tableName); 364 assertEquals(nregions, tableRegions.size()); 365 return tableRegions; 366 } 367 368 private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() { 369 return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); 370 } 371 372 private void collectAssignmentManagerMetrics() { 373 mergeSubmittedCount = mergeProcMetrics.getSubmittedCounter().getCount(); 374 mergeFailedCount = mergeProcMetrics.getFailedCounter().getCount(); 375 376 assignSubmittedCount = assignProcMetrics.getSubmittedCounter().getCount(); 377 assignFailedCount = assignProcMetrics.getFailedCounter().getCount(); 378 unassignSubmittedCount = unassignProcMetrics.getSubmittedCounter().getCount(); 379 unassignFailedCount = unassignProcMetrics.getFailedCounter().getCount(); 380 } 381}