001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.assertProcFailed;
021import static org.junit.jupiter.api.Assertions.assertEquals;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.List;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.MetaTableAccessor;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.Admin;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.Put;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.client.SnapshotDescription;
039import org.apache.hadoop.hbase.client.SnapshotType;
040import org.apache.hadoop.hbase.client.Table;
041import org.apache.hadoop.hbase.client.TableDescriptor;
042import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
043import org.apache.hadoop.hbase.master.RegionState;
044import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
045import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
046import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility;
047import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
048import org.apache.hadoop.hbase.master.procedure.TestSnapshotProcedure;
049import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
050import org.apache.hadoop.hbase.procedure2.ProcedureMetrics;
051import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
052import org.apache.hadoop.hbase.regionserver.HRegion;
053import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
054import org.apache.hadoop.hbase.testclassification.LargeTests;
055import org.apache.hadoop.hbase.testclassification.MasterTests;
056import org.apache.hadoop.hbase.util.Bytes;
057import org.apache.hadoop.hbase.util.CommonFSUtils;
058import org.apache.hadoop.hbase.util.FSUtils;
059import org.apache.hadoop.hbase.util.Threads;
060import org.apache.hadoop.hbase.wal.WALSplitUtil;
061import org.junit.jupiter.api.AfterAll;
062import org.junit.jupiter.api.AfterEach;
063import org.junit.jupiter.api.BeforeAll;
064import org.junit.jupiter.api.BeforeEach;
065import org.junit.jupiter.api.Tag;
066import org.junit.jupiter.api.Test;
067import org.junit.jupiter.api.TestInfo;
068import org.slf4j.Logger;
069import org.slf4j.LoggerFactory;
070
071import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
072import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
073
074@Tag(MasterTests.TAG)
075@Tag(LargeTests.TAG)
076public class TestMergeTableRegionsProcedure {
077
078  private static final Logger LOG = LoggerFactory.getLogger(TestMergeTableRegionsProcedure.class);
079  private String testMethodName;
080
081  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
082
083  private static final int initialRegionCount = 4;
084  private final static byte[] FAMILY = Bytes.toBytes("FAMILY");
085  private static Admin admin;
086
087  private ProcedureMetrics mergeProcMetrics;
088  private ProcedureMetrics assignProcMetrics;
089  private ProcedureMetrics unassignProcMetrics;
090  private long mergeSubmittedCount = 0;
091  private long mergeFailedCount = 0;
092  private long assignSubmittedCount = 0;
093  private long assignFailedCount = 0;
094  private long unassignSubmittedCount = 0;
095  private long unassignFailedCount = 0;
096
097  private static void setupConf(Configuration conf) {
098    // Reduce the maximum attempts to speed up the test
099    conf.setInt("hbase.assignment.maximum.attempts", 3);
100    conf.setInt("hbase.master.maximum.ping.server.attempts", 3);
101    conf.setInt("hbase.master.ping.server.retry.sleep.interval", 1);
102    conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
103    conf.set("hbase.coprocessor.region.classes",
104      RegionServerHostingReplicaSlowOpenCoprocessor.class.getName());
105  }
106
107  @BeforeAll
108  public static void setupCluster() throws Exception {
109    setupConf(UTIL.getConfiguration());
110    UTIL.startMiniCluster(1);
111    admin = UTIL.getAdmin();
112  }
113
114  @AfterAll
115  public static void cleanupTest() throws Exception {
116    UTIL.shutdownMiniCluster();
117  }
118
119  @BeforeEach
120  public void setup() throws Exception {
121    resetProcExecutorTestingKillFlag();
122    MasterProcedureTestingUtility.generateNonceGroup(UTIL.getHBaseCluster().getMaster());
123    MasterProcedureTestingUtility.generateNonce(UTIL.getHBaseCluster().getMaster());
124    // Turn off balancer so it doesn't cut in and mess up our placements.
125    admin.balancerSwitch(false, true);
126    // Turn off the meta scanner so it don't remove parent on us.
127    UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
128    resetProcExecutorTestingKillFlag();
129    AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
130    mergeProcMetrics = am.getAssignmentManagerMetrics().getMergeProcMetrics();
131    assignProcMetrics = am.getAssignmentManagerMetrics().getAssignProcMetrics();
132    unassignProcMetrics = am.getAssignmentManagerMetrics().getUnassignProcMetrics();
133  }
134
135  @BeforeEach
136  public void setTestMethod(TestInfo testInfo) {
137    testMethodName = testInfo.getTestMethod().get().getName();
138  }
139
140  @AfterEach
141  public void tearDown() throws Exception {
142    resetProcExecutorTestingKillFlag();
143    for (TableDescriptor htd : admin.listTableDescriptors()) {
144      LOG.info("Tear down, remove table=" + htd.getTableName());
145      UTIL.deleteTable(htd.getTableName());
146    }
147  }
148
149  private void resetProcExecutorTestingKillFlag() {
150    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
151    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
152    assertTrue(procExec.isRunning(), "expected executor to be running");
153  }
154
155  private int loadARowPerRegion(final Table t, List<RegionInfo> ris) throws IOException {
156    List<Put> puts = new ArrayList<>();
157    for (RegionInfo ri : ris) {
158      Put put = new Put(ri.getStartKey() == null || ri.getStartKey().length == 0
159        ? new byte[] { 'a' }
160        : ri.getStartKey());
161      put.addColumn(HConstants.CATALOG_FAMILY, HConstants.CATALOG_FAMILY,
162        HConstants.CATALOG_FAMILY);
163      puts.add(put);
164    }
165    t.put(puts);
166    return puts.size();
167  }
168
169  /**
170   * This tests two region merges
171   */
172  @Test
173  public void testMergeTwoRegions() throws Exception {
174    final TableName tableName = TableName.valueOf(testMethodName);
175    UTIL.createTable(tableName, new byte[][] { HConstants.CATALOG_FAMILY }, new byte[][] {
176      new byte[] { 'b' }, new byte[] { 'c' }, new byte[] { 'd' }, new byte[] { 'e' } });
177    testMerge(tableName, 2);
178  }
179
180  private void testMerge(TableName tableName, int mergeCount) throws IOException {
181    List<RegionInfo> ris = MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName);
182    int originalRegionCount = ris.size();
183    assertTrue(originalRegionCount > mergeCount);
184    RegionInfo[] regionsToMerge = ris.subList(0, mergeCount).toArray(new RegionInfo[] {});
185    int countOfRowsLoaded = 0;
186    try (Table table = UTIL.getConnection().getTable(tableName)) {
187      countOfRowsLoaded = loadARowPerRegion(table, ris);
188    }
189    assertEquals(countOfRowsLoaded, UTIL.countRows(tableName));
190
191    // collect AM metrics before test
192    collectAssignmentManagerMetrics();
193    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
194    MergeTableRegionsProcedure proc =
195      new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true);
196    long procId = procExec.submitProcedure(proc);
197    ProcedureTestingUtility.waitProcedure(procExec, procId);
198    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
199    MetaTableAccessor.fullScanMetaAndPrint(UTIL.getConnection());
200    assertEquals(originalRegionCount - mergeCount + 1,
201      MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName).size());
202
203    assertEquals(mergeSubmittedCount + 1, mergeProcMetrics.getSubmittedCounter().getCount());
204    assertEquals(mergeFailedCount, mergeProcMetrics.getFailedCounter().getCount());
205    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
206    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
207    assertEquals(unassignSubmittedCount + mergeCount,
208      unassignProcMetrics.getSubmittedCounter().getCount());
209    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
210
211    // Need to get the references cleaned out. Close of region will move them
212    // to archive so disable and reopen just to get rid of references to later
213    // when the catalogjanitor runs, it can do merged region cleanup.
214    admin.disableTable(tableName);
215    admin.enableTable(tableName);
216
217    // Can I purge the merged regions from hbase:meta? Check that all went
218    // well by looking at the merged row up in hbase:meta. It should have no
219    // more mention of the merged regions; they are purged as last step in
220    // the merged regions cleanup.
221    UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(true);
222    UTIL.getHBaseCluster().getMaster().getCatalogJanitor().triggerNow();
223    RegionInfo mergedRegion = proc.getMergedRegion();
224    RegionStateStore regionStateStore =
225      UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStateStore();
226    while (ris != null && ris.get(0) != null && ris.get(1) != null) {
227      ris = regionStateStore.getMergeRegions(mergedRegion);
228      LOG.info("{} {}", Bytes.toStringBinary(mergedRegion.getRegionName()), ris);
229      Threads.sleep(1000);
230    }
231    assertEquals(countOfRowsLoaded, UTIL.countRows(tableName));
232  }
233
234  /**
235   * This tests ten region merges in one go.
236   */
237  @Test
238  public void testMergeTenRegions() throws Exception {
239    final TableName tableName = TableName.valueOf(testMethodName);
240    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
241    UTIL.createMultiRegionTable(tableName, HConstants.CATALOG_FAMILY);
242    testMerge(tableName, 10);
243  }
244
245  /**
246   * This tests two concurrent region merges
247   */
248  @Test
249  public void testMergeRegionsConcurrently() throws Exception {
250    final TableName tableName = TableName.valueOf("testMergeRegionsConcurrently");
251    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
252
253    List<RegionInfo> tableRegions = createTable(tableName);
254
255    RegionInfo[] regionsToMerge1 = new RegionInfo[2];
256    RegionInfo[] regionsToMerge2 = new RegionInfo[2];
257    regionsToMerge1[0] = tableRegions.get(0);
258    regionsToMerge1[1] = tableRegions.get(1);
259    regionsToMerge2[0] = tableRegions.get(2);
260    regionsToMerge2[1] = tableRegions.get(3);
261
262    // collect AM metrics before test
263    collectAssignmentManagerMetrics();
264
265    long procId1 = procExec.submitProcedure(
266      new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge1, true));
267    long procId2 = procExec.submitProcedure(
268      new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge2, true));
269    ProcedureTestingUtility.waitProcedure(procExec, procId1);
270    ProcedureTestingUtility.waitProcedure(procExec, procId2);
271    ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
272    ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);
273    assertRegionCount(tableName, initialRegionCount - 2);
274
275    assertEquals(mergeSubmittedCount + 2, mergeProcMetrics.getSubmittedCounter().getCount());
276    assertEquals(mergeFailedCount, mergeProcMetrics.getFailedCounter().getCount());
277    assertEquals(assignSubmittedCount + 2, assignProcMetrics.getSubmittedCounter().getCount());
278    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
279    assertEquals(unassignSubmittedCount + 4, unassignProcMetrics.getSubmittedCounter().getCount());
280    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
281  }
282
283  @Test
284  public void testRecoveryAndDoubleExecution() throws Exception {
285    final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution");
286    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
287
288    List<RegionInfo> tableRegions = createTable(tableName);
289
290    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
291    ProcedureTestingUtility.setKillIfHasParent(procExec, false);
292    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
293
294    RegionInfo[] regionsToMerge = new RegionInfo[2];
295    regionsToMerge[0] = tableRegions.get(0);
296    regionsToMerge[1] = tableRegions.get(1);
297
298    long procId = procExec.submitProcedure(
299      new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true));
300
301    // Restart the executor and execute the step twice
302    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
303    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
304
305    assertRegionCount(tableName, initialRegionCount - 1);
306  }
307
308  @Test
309  public void testRollbackAndDoubleExecution() throws Exception {
310    final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
311    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
312
313    List<RegionInfo> tableRegions = createTable(tableName);
314
315    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
316    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
317
318    RegionInfo[] regionsToMerge = new RegionInfo[2];
319    regionsToMerge[0] = tableRegions.get(0);
320    regionsToMerge[1] = tableRegions.get(1);
321
322    long procId = procExec.submitProcedure(
323      new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true));
324
325    // Failing before MERGE_TABLE_REGIONS_UPDATE_META we should trigger the rollback
326    // NOTE: the 8 (number of MERGE_TABLE_REGIONS_UPDATE_META step) is
327    // hardcoded, so you have to look at this test at least once when you add a new step.
328    int lastStep = 8;
329    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep, true);
330    assertEquals(initialRegionCount, UTIL.getAdmin().getRegions(tableName).size());
331    UTIL.waitUntilAllRegionsAssigned(tableName);
332    List<HRegion> regions = UTIL.getMiniHBaseCluster().getRegions(tableName);
333    assertEquals(initialRegionCount, regions.size());
334  }
335
336  /**
337   * HBASE-30334 repro. Plant a stale recovered.edits file on a parent region so that
338   * MERGE_TABLE_REGIONS_CHECK_CLOSED_REGIONS throws. After rollback the parents MUST be OPEN. If
339   * they stay stuck (e.g. CLOSED / MERGING), the rollback path is broken.
340   */
341  @Test
342  public void testRollbackReopensParentsAfterCheckClosedRegionsFailure() throws Exception {
343    final TableName tableName = TableName.valueOf(testMethodName);
344    UTIL.createTable(tableName, new byte[][] { HConstants.CATALOG_FAMILY },
345      new byte[][] { new byte[] { 'b' } });
346    UTIL.waitUntilAllRegionsAssigned(tableName);
347
348    List<RegionInfo> ris = MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName);
349    assertEquals(2, ris.size());
350    RegionInfo[] regionsToMerge = new RegionInfo[] { ris.get(0), ris.get(1) };
351
352    Configuration conf = UTIL.getConfiguration();
353    Path regionDir =
354      FSUtils.getRegionDirFromRootDir(CommonFSUtils.getRootDir(conf), regionsToMerge[0]);
355    Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regionDir);
356    FileSystem fs = CommonFSUtils.getRootDirFileSystem(conf);
357    fs.mkdirs(recoveredEditsDir);
358    Path staleFile = new Path(recoveredEditsDir, "0000000000000000001");
359    fs.createNewFile(staleFile);
360    assertTrue(WALSplitUtil.hasRecoveredEdits(conf, regionsToMerge[0]),
361      "stale recovered.edits file must be visible");
362
363    AssignmentManager am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
364    LOG.info("HBASE-30334-DEBUG pre-merge: {} state={} / {} state={}",
365      regionsToMerge[0].getEncodedName(),
366      am.getRegionStates().getRegionStateNode(regionsToMerge[0]).getState(),
367      regionsToMerge[1].getEncodedName(),
368      am.getRegionStates().getRegionStateNode(regionsToMerge[1]).getState());
369
370    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
371    MergeTableRegionsProcedure proc =
372      new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true);
373    long procId = procExec.submitProcedure(proc);
374    ProcedureTestingUtility.waitProcedure(procExec, procId);
375
376    RegionState.State s0Post =
377      am.getRegionStates().getRegionStateNode(regionsToMerge[0]).getState();
378    RegionState.State s1Post =
379      am.getRegionStates().getRegionStateNode(regionsToMerge[1]).getState();
380    LOG.info("HBASE-30334-DEBUG post-rollback: {} state={} / {} state={}",
381      regionsToMerge[0].getEncodedName(), s0Post, regionsToMerge[1].getEncodedName(), s1Post);
382
383    ProcedureTestingUtility.assertProcFailed(procExec, procId);
384
385    fs.delete(staleFile, false);
386
387    UTIL.waitFor(30_000, 500, () -> {
388      RegionState.State a = am.getRegionStates().getRegionStateNode(regionsToMerge[0]).getState();
389      RegionState.State b = am.getRegionStates().getRegionStateNode(regionsToMerge[1]).getState();
390      return a == RegionState.State.OPEN && b == RegionState.State.OPEN;
391    });
392  }
393
394  @Test
395  public void testMergeWithoutPONR() throws Exception {
396    final TableName tableName = TableName.valueOf("testMergeWithoutPONR");
397    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
398
399    List<RegionInfo> tableRegions = createTable(tableName);
400
401    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
402    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
403
404    RegionInfo[] regionsToMerge = new RegionInfo[2];
405    regionsToMerge[0] = tableRegions.get(0);
406    regionsToMerge[1] = tableRegions.get(1);
407
408    long procId = procExec.submitProcedure(
409      new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true));
410
411    // Execute until step 9 of split procedure
412    // NOTE: step 9 is after step MERGE_TABLE_REGIONS_UPDATE_META
413    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, 9, false);
414
415    // Unset Toggle Kill and make ProcExec work correctly
416    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
417    MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
418    ProcedureTestingUtility.waitProcedure(procExec, procId);
419
420    assertRegionCount(tableName, initialRegionCount - 1);
421  }
422
423  @Test
424  public void testMergingRegionWhileTakingSnapshot() throws Exception {
425    final TableName tableName = TableName.valueOf("testMergingRegionWhileTakingSnapshot");
426    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
427
428    List<RegionInfo> tableRegions = createTable(tableName);
429
430    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
431
432    SnapshotDescription snapshot =
433      new SnapshotDescription("SnapshotProcedureTest", tableName, SnapshotType.FLUSH);
434    SnapshotProtos.SnapshotDescription snapshotProto =
435      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot);
436    snapshotProto = SnapshotDescriptionUtils.validate(snapshotProto,
437      UTIL.getHBaseCluster().getMaster().getConfiguration());
438    long snapshotProcId = procExec.submitProcedure(
439      new TestSnapshotProcedure.DelaySnapshotProcedure(procExec.getEnvironment(), snapshotProto));
440    UTIL.getHBaseCluster().getMaster().getSnapshotManager().registerSnapshotProcedure(snapshotProto,
441      snapshotProcId);
442
443    RegionInfo[] regionsToMerge = new RegionInfo[2];
444    regionsToMerge[0] = tableRegions.get(0);
445    regionsToMerge[1] = tableRegions.get(1);
446
447    long mergeProcId = procExec.submitProcedure(
448      new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true));
449
450    ProcedureTestingUtility
451      .waitProcedure(UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(), mergeProcId);
452    ProcedureTestingUtility.waitProcedure(
453      UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(), snapshotProcId);
454
455    assertProcFailed(procExec, mergeProcId);
456    assertEquals(initialRegionCount, UTIL.getAdmin().getRegions(tableName).size());
457  }
458
459  @Test
460  public void testMergeDetectsModifyTableProcedure() throws Exception {
461    final TableName tableName = TableName.valueOf(testMethodName);
462    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
463
464    List<RegionInfo> regions = createTable(tableName);
465
466    RegionServerHostingReplicaSlowOpenCoprocessor.slowDownReplicaOpen = true;
467    TableDescriptor td = TableDescriptorBuilder.newBuilder(admin.getDescriptor(tableName))
468      .setRegionReplication(2).build();
469    long modifyProcId =
470      procExec.submitProcedure(new ModifyTableProcedure(procExec.getEnvironment(), td));
471
472    // Merge regions of the table, the MergeTableRegionsProcedure will fail because there is a
473    // ModifyTableProcedure in progress
474    MergeTableRegionsProcedure mergeProcedure = new MergeTableRegionsProcedure(
475      procExec.getEnvironment(), regions.toArray(new RegionInfo[0]), false);
476    long mergeProcId = procExec.submitProcedure(mergeProcedure);
477    ProcedureTestingUtility.waitProcedure(procExec, mergeProcId);
478    ProcedureTestingUtility.assertProcFailed(procExec, mergeProcId);
479
480    RegionServerHostingReplicaSlowOpenCoprocessor.slowDownReplicaOpen = false;
481    ProcedureTestingUtility.waitProcedure(procExec, modifyProcId);
482    ProcedureTestingUtility.assertProcNotFailed(procExec, modifyProcId);
483  }
484
485  private List<RegionInfo> createTable(final TableName tableName) throws Exception {
486    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
487      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build();
488    byte[][] splitRows = new byte[initialRegionCount - 1][];
489    for (int i = 0; i < splitRows.length; ++i) {
490      splitRows[i] = Bytes.toBytes(String.format("%d", i));
491    }
492    admin.createTable(desc, splitRows);
493    return assertRegionCount(tableName, initialRegionCount);
494  }
495
496  public List<RegionInfo> assertRegionCount(final TableName tableName, final int nregions)
497    throws Exception {
498    UTIL.waitUntilNoRegionsInTransition();
499    List<RegionInfo> tableRegions = admin.getRegions(tableName);
500    assertEquals(nregions, tableRegions.size());
501    return tableRegions;
502  }
503
504  private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
505    return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
506  }
507
508  private void collectAssignmentManagerMetrics() {
509    mergeSubmittedCount = mergeProcMetrics.getSubmittedCounter().getCount();
510    mergeFailedCount = mergeProcMetrics.getFailedCounter().getCount();
511
512    assignSubmittedCount = assignProcMetrics.getSubmittedCounter().getCount();
513    assignFailedCount = assignProcMetrics.getFailedCounter().getCount();
514    unassignSubmittedCount = unassignProcMetrics.getSubmittedCounter().getCount();
515    unassignFailedCount = unassignProcMetrics.getFailedCounter().getCount();
516  }
517}