001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.List;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.MetaTableAccessor;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.Admin;
033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
034import org.apache.hadoop.hbase.client.Put;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.client.Table;
037import org.apache.hadoop.hbase.client.TableDescriptor;
038import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
039import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
040import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
041import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility;
042import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
043import org.apache.hadoop.hbase.procedure2.ProcedureMetrics;
044import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
045import org.apache.hadoop.hbase.regionserver.HRegion;
046import org.apache.hadoop.hbase.testclassification.MasterTests;
047import org.apache.hadoop.hbase.testclassification.MediumTests;
048import org.apache.hadoop.hbase.util.Bytes;
049import org.apache.hadoop.hbase.util.Threads;
050import org.junit.After;
051import org.junit.AfterClass;
052import org.junit.Before;
053import org.junit.BeforeClass;
054import org.junit.ClassRule;
055import org.junit.Rule;
056import org.junit.Test;
057import org.junit.experimental.categories.Category;
058import org.junit.rules.TestName;
059import org.slf4j.Logger;
060import org.slf4j.LoggerFactory;
061
062@Category({MasterTests.class, MediumTests.class})
063public class TestMergeTableRegionsProcedure {
064
065  @ClassRule
066  public static final HBaseClassTestRule CLASS_RULE =
067      HBaseClassTestRule.forClass(TestMergeTableRegionsProcedure.class);
068
069  private static final Logger LOG = LoggerFactory.getLogger(TestMergeTableRegionsProcedure.class);
070  @Rule
071  public final TestName name = new TestName();
072
073  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
074
075  private static final int initialRegionCount = 4;
076  private final static byte[] FAMILY = Bytes.toBytes("FAMILY");
077  private static Admin admin;
078
079  private ProcedureMetrics mergeProcMetrics;
080  private ProcedureMetrics assignProcMetrics;
081  private ProcedureMetrics unassignProcMetrics;
082  private long mergeSubmittedCount = 0;
083  private long mergeFailedCount = 0;
084  private long assignSubmittedCount = 0;
085  private long assignFailedCount = 0;
086  private long unassignSubmittedCount = 0;
087  private long unassignFailedCount = 0;
088
089  private static void setupConf(Configuration conf) {
090    // Reduce the maximum attempts to speed up the test
091    conf.setInt("hbase.assignment.maximum.attempts", 3);
092    conf.setInt("hbase.master.maximum.ping.server.attempts", 3);
093    conf.setInt("hbase.master.ping.server.retry.sleep.interval", 1);
094    conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
095  }
096
097  @BeforeClass
098  public static void setupCluster() throws Exception {
099    setupConf(UTIL.getConfiguration());
100    UTIL.startMiniCluster(1);
101    admin = UTIL.getAdmin();
102  }
103
104  @AfterClass
105  public static void cleanupTest() throws Exception {
106    UTIL.shutdownMiniCluster();
107  }
108
109  @Before
110  public void setup() throws Exception {
111    resetProcExecutorTestingKillFlag();
112    MasterProcedureTestingUtility.generateNonceGroup(UTIL.getHBaseCluster().getMaster());
113    MasterProcedureTestingUtility.generateNonce(UTIL.getHBaseCluster().getMaster());
114    // Turn off balancer so it doesn't cut in and mess up our placements.
115    admin.balancerSwitch(false, true);
116    // Turn off the meta scanner so it don't remove parent on us.
117    UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
118    resetProcExecutorTestingKillFlag();
119    AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
120    mergeProcMetrics = am.getAssignmentManagerMetrics().getMergeProcMetrics();
121    assignProcMetrics = am.getAssignmentManagerMetrics().getAssignProcMetrics();
122    unassignProcMetrics = am.getAssignmentManagerMetrics().getUnassignProcMetrics();
123  }
124
125  @After
126  public void tearDown() throws Exception {
127    resetProcExecutorTestingKillFlag();
128    for (TableDescriptor htd: admin.listTableDescriptors()) {
129      LOG.info("Tear down, remove table=" + htd.getTableName());
130      UTIL.deleteTable(htd.getTableName());
131    }
132  }
133
134  private void resetProcExecutorTestingKillFlag() {
135    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
136    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
137    assertTrue("expected executor to be running", procExec.isRunning());
138  }
139
140  private int loadARowPerRegion(final Table t, List<RegionInfo> ris)
141      throws IOException {
142    List<Put> puts = new ArrayList<>();
143    for (RegionInfo ri: ris) {
144      Put put = new Put(ri.getStartKey() == null || ri.getStartKey().length == 0?
145          new byte [] {'a'}: ri.getStartKey());
146      put.addColumn(HConstants.CATALOG_FAMILY, HConstants.CATALOG_FAMILY,
147          HConstants.CATALOG_FAMILY);
148      puts.add(put);
149    }
150    t.put(puts);
151    return puts.size();
152  }
153
154
155  /**
156   * This tests two region merges
157   */
158  @Test
159  public void testMergeTwoRegions() throws Exception {
160    final TableName tableName = TableName.valueOf(this.name.getMethodName());
161    UTIL.createTable(tableName, new byte[][]{HConstants.CATALOG_FAMILY},
162        new byte[][]{new byte[]{'b'}, new byte[]{'c'}, new byte[]{'d'}, new byte[]{'e'}});
163    testMerge(tableName, 2);
164  }
165
166  private void testMerge(TableName tableName, int mergeCount) throws IOException {
167    List<RegionInfo> ris = MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName);
168    int originalRegionCount = ris.size();
169    assertTrue(originalRegionCount > mergeCount);
170    RegionInfo[] regionsToMerge = ris.subList(0, mergeCount).toArray(new RegionInfo [] {});
171    int countOfRowsLoaded = 0;
172    try (Table table = UTIL.getConnection().getTable(tableName)) {
173      countOfRowsLoaded = loadARowPerRegion(table, ris);
174    }
175    assertEquals(countOfRowsLoaded, UTIL.countRows(tableName));
176
177    // collect AM metrics before test
178    collectAssignmentManagerMetrics();
179    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
180    MergeTableRegionsProcedure proc =
181        new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true);
182    long procId = procExec.submitProcedure(proc);
183    ProcedureTestingUtility.waitProcedure(procExec, procId);
184    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
185    MetaTableAccessor.fullScanMetaAndPrint(UTIL.getConnection());
186    assertEquals(originalRegionCount - mergeCount + 1,
187        MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName).size());
188
189    assertEquals(mergeSubmittedCount + 1, mergeProcMetrics.getSubmittedCounter().getCount());
190    assertEquals(mergeFailedCount, mergeProcMetrics.getFailedCounter().getCount());
191    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
192    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
193    assertEquals(unassignSubmittedCount + mergeCount,
194        unassignProcMetrics.getSubmittedCounter().getCount());
195    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
196
197    // Need to get the references cleaned out. Close of region will move them
198    // to archive so disable and reopen just to get rid of references to later
199    // when the catalogjanitor runs, it can do merged region cleanup.
200    admin.disableTable(tableName);
201    admin.enableTable(tableName);
202
203    // Can I purge the merged regions from hbase:meta? Check that all went
204    // well by looking at the merged row up in hbase:meta. It should have no
205    // more mention of the merged regions; they are purged as last step in
206    // the merged regions cleanup.
207    UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(true);
208    UTIL.getHBaseCluster().getMaster().getCatalogJanitor().triggerNow();
209    byte [] mergedRegion = proc.getMergedRegion().getRegionName();
210    while (ris != null && ris.get(0) != null && ris.get(1) != null) {
211      ris = MetaTableAccessor.getMergeRegions(UTIL.getConnection(), mergedRegion);
212      LOG.info("{} {}", Bytes.toStringBinary(mergedRegion), ris);
213      Threads.sleep(1000);
214    }
215    assertEquals(countOfRowsLoaded, UTIL.countRows(tableName));
216  }
217
218  /**
219   * This tests ten region merges in one go.
220   */
221  @Test
222  public void testMergeTenRegions() throws Exception {
223    final TableName tableName = TableName.valueOf(this.name.getMethodName());
224    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
225    UTIL.createMultiRegionTable(tableName, HConstants.CATALOG_FAMILY);
226    testMerge(tableName, 10);
227  }
228
229  /**
230   * This tests two concurrent region merges
231   */
232  @Test
233  public void testMergeRegionsConcurrently() throws Exception {
234    final TableName tableName = TableName.valueOf("testMergeRegionsConcurrently");
235    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
236
237    List<RegionInfo> tableRegions = createTable(tableName);
238
239    RegionInfo[] regionsToMerge1 = new RegionInfo[2];
240    RegionInfo[] regionsToMerge2 = new RegionInfo[2];
241    regionsToMerge1[0] = tableRegions.get(0);
242    regionsToMerge1[1] = tableRegions.get(1);
243    regionsToMerge2[0] = tableRegions.get(2);
244    regionsToMerge2[1] = tableRegions.get(3);
245
246    // collect AM metrics before test
247    collectAssignmentManagerMetrics();
248
249    long procId1 = procExec.submitProcedure(new MergeTableRegionsProcedure(
250      procExec.getEnvironment(), regionsToMerge1, true));
251    long procId2 = procExec.submitProcedure(new MergeTableRegionsProcedure(
252      procExec.getEnvironment(), regionsToMerge2, true));
253    ProcedureTestingUtility.waitProcedure(procExec, procId1);
254    ProcedureTestingUtility.waitProcedure(procExec, procId2);
255    ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
256    ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);
257    assertRegionCount(tableName, initialRegionCount - 2);
258
259    assertEquals(mergeSubmittedCount + 2, mergeProcMetrics.getSubmittedCounter().getCount());
260    assertEquals(mergeFailedCount, mergeProcMetrics.getFailedCounter().getCount());
261    assertEquals(assignSubmittedCount + 2, assignProcMetrics.getSubmittedCounter().getCount());
262    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
263    assertEquals(unassignSubmittedCount + 4, unassignProcMetrics.getSubmittedCounter().getCount());
264    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
265  }
266
267  @Test
268  public void testRecoveryAndDoubleExecution() throws Exception {
269    final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution");
270    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
271
272    List<RegionInfo> tableRegions = createTable(tableName);
273
274    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
275    ProcedureTestingUtility.setKillIfHasParent(procExec, false);
276    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
277
278    RegionInfo[] regionsToMerge = new RegionInfo[2];
279    regionsToMerge[0] = tableRegions.get(0);
280    regionsToMerge[1] = tableRegions.get(1);
281
282    long procId = procExec.submitProcedure(
283      new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true));
284
285    // Restart the executor and execute the step twice
286    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
287    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
288
289    assertRegionCount(tableName, initialRegionCount - 1);
290  }
291
292  @Test
293  public void testRollbackAndDoubleExecution() throws Exception {
294    final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
295    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
296
297    List<RegionInfo> tableRegions = createTable(tableName);
298
299    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
300    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
301
302    RegionInfo[] regionsToMerge = new RegionInfo[2];
303    regionsToMerge[0] = tableRegions.get(0);
304    regionsToMerge[1] = tableRegions.get(1);
305
306    long procId = procExec.submitProcedure(
307      new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true));
308
309    // Failing before MERGE_TABLE_REGIONS_UPDATE_META we should trigger the rollback
310    // NOTE: the 8 (number of MERGE_TABLE_REGIONS_UPDATE_META step) is
311    // hardcoded, so you have to look at this test at least once when you add a new step.
312    int lastStep = 8;
313    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep, true);
314    assertEquals(initialRegionCount, UTIL.getAdmin().getRegions(tableName).size());
315    UTIL.waitUntilAllRegionsAssigned(tableName);
316    List<HRegion> regions = UTIL.getMiniHBaseCluster().getRegions(tableName);
317    assertEquals(initialRegionCount, regions.size());
318  }
319
320  @Test
321  public void testMergeWithoutPONR() throws Exception {
322    final TableName tableName = TableName.valueOf("testMergeWithoutPONR");
323    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
324
325    List<RegionInfo> tableRegions = createTable(tableName);
326
327    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
328    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
329
330    RegionInfo[] regionsToMerge = new RegionInfo[2];
331    regionsToMerge[0] = tableRegions.get(0);
332    regionsToMerge[1] = tableRegions.get(1);
333
334    long procId = procExec.submitProcedure(
335      new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true));
336
337    // Execute until step 9 of split procedure
338    // NOTE: step 9 is after step MERGE_TABLE_REGIONS_UPDATE_META
339    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, 9, false);
340
341    // Unset Toggle Kill and make ProcExec work correctly
342    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
343    MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
344    ProcedureTestingUtility.waitProcedure(procExec, procId);
345
346    assertRegionCount(tableName, initialRegionCount - 1);
347  }
348
349  private List<RegionInfo> createTable(final TableName tableName) throws Exception {
350    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
351      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build();
352    byte[][] splitRows = new byte[initialRegionCount - 1][];
353    for (int i = 0; i < splitRows.length; ++i) {
354      splitRows[i] = Bytes.toBytes(String.format("%d", i));
355    }
356    admin.createTable(desc, splitRows);
357    return assertRegionCount(tableName, initialRegionCount);
358  }
359
360  public List<RegionInfo> assertRegionCount(final TableName tableName, final int nregions)
361      throws Exception {
362    UTIL.waitUntilNoRegionsInTransition();
363    List<RegionInfo> tableRegions = admin.getRegions(tableName);
364    assertEquals(nregions, tableRegions.size());
365    return tableRegions;
366  }
367
368  private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
369    return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
370  }
371
372  private void collectAssignmentManagerMetrics() {
373    mergeSubmittedCount = mergeProcMetrics.getSubmittedCounter().getCount();
374    mergeFailedCount = mergeProcMetrics.getFailedCounter().getCount();
375
376    assignSubmittedCount = assignProcMetrics.getSubmittedCounter().getCount();
377    assignFailedCount = assignProcMetrics.getFailedCounter().getCount();
378    unassignSubmittedCount = unassignProcMetrics.getSubmittedCounter().getCount();
379    unassignFailedCount = unassignProcMetrics.getFailedCounter().getCount();
380  }
381}