001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.List;
028import java.util.Objects;
029import java.util.concurrent.TimeUnit;
030import java.util.concurrent.atomic.AtomicBoolean;
031import org.apache.commons.lang3.RandomUtils;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.fs.FileSystem;
034import org.apache.hadoop.fs.Path;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.HBaseTestingUtility;
037import org.apache.hadoop.hbase.MetaTableAccessor;
038import org.apache.hadoop.hbase.MiniHBaseCluster;
039import org.apache.hadoop.hbase.ServerName;
040import org.apache.hadoop.hbase.StartMiniClusterOption;
041import org.apache.hadoop.hbase.TableName;
042import org.apache.hadoop.hbase.UnknownRegionException;
043import org.apache.hadoop.hbase.client.Admin;
044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
045import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
046import org.apache.hadoop.hbase.client.Put;
047import org.apache.hadoop.hbase.client.RegionInfo;
048import org.apache.hadoop.hbase.client.RegionReplicaUtil;
049import org.apache.hadoop.hbase.client.Result;
050import org.apache.hadoop.hbase.client.ResultScanner;
051import org.apache.hadoop.hbase.client.Scan;
052import org.apache.hadoop.hbase.client.Table;
053import org.apache.hadoop.hbase.client.TableDescriptor;
054import org.apache.hadoop.hbase.exceptions.MergeRegionException;
055import org.apache.hadoop.hbase.master.HMaster;
056import org.apache.hadoop.hbase.master.MasterRpcServices;
057import org.apache.hadoop.hbase.master.RegionState;
058import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
059import org.apache.hadoop.hbase.master.assignment.RegionStates;
060import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
061import org.apache.hadoop.hbase.testclassification.LargeTests;
062import org.apache.hadoop.hbase.testclassification.RegionServerTests;
063import org.apache.hadoop.hbase.util.Bytes;
064import org.apache.hadoop.hbase.util.FSUtils;
065import org.apache.hadoop.hbase.util.FutureUtils;
066import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
067import org.apache.hadoop.hbase.util.Pair;
068import org.apache.hadoop.hbase.util.PairOfSameType;
069import org.apache.hadoop.util.StringUtils;
070import org.apache.zookeeper.KeeperException;
071import org.junit.AfterClass;
072import org.junit.BeforeClass;
073import org.junit.ClassRule;
074import org.junit.Rule;
075import org.junit.Test;
076import org.junit.experimental.categories.Category;
077import org.junit.rules.TestName;
078import org.slf4j.Logger;
079import org.slf4j.LoggerFactory;
080
081import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
082import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
083import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
084
085import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
086import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
087import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
088
089@Category({RegionServerTests.class, LargeTests.class})
090public class TestRegionMergeTransactionOnCluster {
091
092  @ClassRule
093  public static final HBaseClassTestRule CLASS_RULE =
094      HBaseClassTestRule.forClass(TestRegionMergeTransactionOnCluster.class);
095
096  private static final Logger LOG =
097      LoggerFactory.getLogger(TestRegionMergeTransactionOnCluster.class);
098
099  @Rule public TestName name = new TestName();
100
101  private static final int NB_SERVERS = 3;
102
103  private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
104  private static final byte[] QUALIFIER = Bytes.toBytes("q");
105
106  private static byte[] ROW = Bytes.toBytes("testRow");
107  private static final int INITIAL_REGION_NUM = 10;
108  private static final int ROWSIZE = 200;
109  private static byte[][] ROWS = makeN(ROW, ROWSIZE);
110
111  private static int waitTime = 60 * 1000;
112
113  static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
114
115  private static HMaster MASTER;
116  private static Admin ADMIN;
117
118  @BeforeClass
119  public static void beforeAllTests() throws Exception {
120    // Start a cluster
121    StartMiniClusterOption option = StartMiniClusterOption.builder()
122        .masterClass(MyMaster.class).numRegionServers(NB_SERVERS).numDataNodes(NB_SERVERS).build();
123    TEST_UTIL.startMiniCluster(option);
124    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
125    MASTER = cluster.getMaster();
126    MASTER.balanceSwitch(false);
127    ADMIN = TEST_UTIL.getConnection().getAdmin();
128  }
129
130  @AfterClass
131  public static void afterAllTests() throws Exception {
132    TEST_UTIL.shutdownMiniCluster();
133    if (ADMIN != null) ADMIN.close();
134  }
135
136  @Test
137  public void testWholesomeMerge() throws Exception {
138    LOG.info("Starting " + name.getMethodName());
139    final TableName tableName = TableName.valueOf(name.getMethodName());
140
141    // Create table and load data.
142    Table table = createTableAndLoadData(MASTER, tableName);
143    // Merge 1st and 2nd region
144    mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1,
145        INITIAL_REGION_NUM - 1);
146
147    // Merge 2nd and 3th region
148    PairOfSameType<RegionInfo> mergedRegions =
149      mergeRegionsAndVerifyRegionNum(MASTER, tableName, 1, 2,
150        INITIAL_REGION_NUM - 2);
151
152    verifyRowCount(table, ROWSIZE);
153
154    // Randomly choose one of the two merged regions
155    RegionInfo hri = RandomUtils.nextBoolean() ?
156      mergedRegions.getFirst() : mergedRegions.getSecond();
157    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
158    AssignmentManager am = cluster.getMaster().getAssignmentManager();
159    RegionStates regionStates = am.getRegionStates();
160
161    // We should not be able to assign it again
162    am.assign(hri);
163    assertFalse("Merged region can't be assigned",
164      regionStates.isRegionInTransition(hri));
165
166    // We should not be able to unassign it either
167    am.unassign(hri);
168    assertFalse("Merged region can't be unassigned",
169      regionStates.isRegionInTransition(hri));
170
171    table.close();
172  }
173
174  /**
175   * Not really restarting the master. Simulate it by clear of new region
176   * state since it is not persisted, will be lost after master restarts.
177   */
178  @Test
179  public void testMergeAndRestartingMaster() throws Exception {
180    final TableName tableName = TableName.valueOf(name.getMethodName());
181
182    // Create table and load data.
183    Table table = createTableAndLoadData(MASTER, tableName);
184
185    try {
186      MyMasterRpcServices.enabled.set(true);
187
188      // Merge 1st and 2nd region
189      mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1, INITIAL_REGION_NUM - 1);
190    } finally {
191      MyMasterRpcServices.enabled.set(false);
192    }
193
194    table.close();
195  }
196
197  @Test
198  public void testCleanMergeReference() throws Exception {
199    LOG.info("Starting " + name.getMethodName());
200    ADMIN.enableCatalogJanitor(false);
201    try {
202      final TableName tableName = TableName.valueOf(name.getMethodName());
203      // Create table and load data.
204      Table table = createTableAndLoadData(MASTER, tableName);
205      // Merge 1st and 2nd region
206      mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1, INITIAL_REGION_NUM - 1);
207      verifyRowCount(table, ROWSIZE);
208      table.close();
209
210      List<Pair<RegionInfo, ServerName>> tableRegions = MetaTableAccessor
211          .getTableRegionsAndLocations(MASTER.getConnection(), tableName);
212      RegionInfo mergedRegionInfo = tableRegions.get(0).getFirst();
213      TableDescriptor tableDescriptor = MASTER.getTableDescriptors().get(
214          tableName);
215      Result mergedRegionResult = MetaTableAccessor.getRegionResult(
216        MASTER.getConnection(), mergedRegionInfo.getRegionName());
217
218      // contains merge reference in META
219      assertTrue(MetaTableAccessor.hasMergeRegions(mergedRegionResult.rawCells()));
220
221      // merging regions' directory are in the file system all the same
222      List<RegionInfo> p = MetaTableAccessor.getMergeRegions(mergedRegionResult.rawCells());
223      RegionInfo regionA = p.get(0);
224      RegionInfo regionB = p.get(1);
225      FileSystem fs = MASTER.getMasterFileSystem().getFileSystem();
226      Path rootDir = MASTER.getMasterFileSystem().getRootDir();
227
228      Path tabledir = FSUtils.getTableDir(rootDir, mergedRegionInfo.getTable());
229      Path regionAdir = new Path(tabledir, regionA.getEncodedName());
230      Path regionBdir = new Path(tabledir, regionB.getEncodedName());
231      assertTrue(fs.exists(regionAdir));
232      assertTrue(fs.exists(regionBdir));
233
234      ColumnFamilyDescriptor[] columnFamilies = tableDescriptor.getColumnFamilies();
235      HRegionFileSystem hrfs = new HRegionFileSystem(
236        TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo);
237      int count = 0;
238      for(ColumnFamilyDescriptor colFamily : columnFamilies) {
239        count += hrfs.getStoreFiles(colFamily.getName()).size();
240      }
241      ADMIN.compactRegion(mergedRegionInfo.getRegionName());
242      // clean up the merged region store files
243      // wait until merged region have reference file
244      long timeout = System.currentTimeMillis() + waitTime;
245      int newcount = 0;
246      while (System.currentTimeMillis() < timeout) {
247        for(ColumnFamilyDescriptor colFamily : columnFamilies) {
248          newcount += hrfs.getStoreFiles(colFamily.getName()).size();
249        }
250        if(newcount > count) {
251          break;
252        }
253        Thread.sleep(50);
254      }
255      assertTrue(newcount > count);
256      List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
257          .getRegionServerThreads();
258      for (RegionServerThread rs : regionServerThreads) {
259        CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null,
260            rs.getRegionServer(), false);
261        cleaner.chore();
262        Thread.sleep(1000);
263      }
264      while (System.currentTimeMillis() < timeout) {
265        int newcount1 = 0;
266        for(ColumnFamilyDescriptor colFamily : columnFamilies) {
267          newcount1 += hrfs.getStoreFiles(colFamily.getName()).size();
268        }
269        if(newcount1 <= 1) {
270          break;
271        }
272        Thread.sleep(50);
273      }
274      // run CatalogJanitor to clean merge references in hbase:meta and archive the
275      // files of merging regions
276      int cleaned = 0;
277      while (cleaned == 0) {
278        cleaned = ADMIN.runCatalogScan();
279        LOG.debug("catalog janitor returned " + cleaned);
280        Thread.sleep(50);
281        // Cleanup is async so wait till all procedures are done running.
282        ProcedureTestingUtility.waitNoProcedureRunning(
283            TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor());
284      }
285      assertFalse(regionAdir.toString(), fs.exists(regionAdir));
286      assertFalse(regionBdir.toString(), fs.exists(regionBdir));
287      assertTrue(cleaned > 0);
288
289      mergedRegionResult = MetaTableAccessor.getRegionResult(
290        TEST_UTIL.getConnection(), mergedRegionInfo.getRegionName());
291      assertFalse(MetaTableAccessor.hasMergeRegions(mergedRegionResult.rawCells()));
292    } finally {
293      ADMIN.enableCatalogJanitor(true);
294    }
295  }
296
297  /**
298   * This test tests 1, merging region not online;
299   * 2, merging same two regions; 3, merging unknown regions.
300   * They are in one test case so that we don't have to create
301   * many tables, and these tests are simple.
302   */
303  @Test
304  public void testMerge() throws Exception {
305    LOG.info("Starting " + name.getMethodName());
306    final TableName tableName = TableName.valueOf(name.getMethodName());
307    final Admin admin = TEST_UTIL.getAdmin();
308    final int syncWaitTimeout = 10 * 60000; // 10min
309
310    try {
311      // Create table and load data.
312      Table table = createTableAndLoadData(MASTER, tableName);
313      AssignmentManager am = MASTER.getAssignmentManager();
314      List<RegionInfo> regions = am.getRegionStates().getRegionsOfTable(tableName);
315      // Fake offline one region
316      RegionInfo a = regions.get(0);
317      RegionInfo b = regions.get(1);
318      am.unassign(b);
319      am.offlineRegion(b);
320      try {
321        // Merge offline region. Region a is offline here
322        admin.mergeRegionsAsync(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false)
323                .get(syncWaitTimeout, TimeUnit.MILLISECONDS);
324        fail("Offline regions should not be able to merge");
325      } catch (DoNotRetryRegionException ie) {
326        System.out.println(ie);
327        assertTrue(ie instanceof MergeRegionException);
328      }
329
330      try {
331        // Merge the same region: b and b.
332        FutureUtils
333          .get(admin.mergeRegionsAsync(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true));
334        fail("A region should not be able to merge with itself, even forcfully");
335      } catch (IOException ie) {
336        assertTrue("Exception should mention regions not online",
337          StringUtils.stringifyException(ie).contains("region to itself")
338            && ie instanceof MergeRegionException);
339      }
340
341      try {
342        // Merge unknown regions
343        admin.mergeRegionsAsync(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true);
344        fail("Unknown region could not be merged");
345      } catch (IOException ie) {
346        assertTrue("UnknownRegionException should be thrown",
347          ie instanceof UnknownRegionException);
348      }
349      table.close();
350    } finally {
351      TEST_UTIL.deleteTable(tableName);
352    }
353  }
354
355  @Test
356  public void testMergeWithReplicas() throws Exception {
357    final TableName tableName = TableName.valueOf(name.getMethodName());
358    // Create table and load data.
359    createTableAndLoadData(MASTER, tableName, 5, 2);
360    List<Pair<RegionInfo, ServerName>> initialRegionToServers =
361        MetaTableAccessor.getTableRegionsAndLocations(
362            TEST_UTIL.getConnection(), tableName);
363    // Merge 1st and 2nd region
364    PairOfSameType<RegionInfo> mergedRegions = mergeRegionsAndVerifyRegionNum(MASTER, tableName,
365        0, 2, 5 * 2 - 2);
366    List<Pair<RegionInfo, ServerName>> currentRegionToServers =
367        MetaTableAccessor.getTableRegionsAndLocations(
368            TEST_UTIL.getConnection(), tableName);
369    List<RegionInfo> initialRegions = new ArrayList<>();
370    for (Pair<RegionInfo, ServerName> p : initialRegionToServers) {
371      initialRegions.add(p.getFirst());
372    }
373    List<RegionInfo> currentRegions = new ArrayList<>();
374    for (Pair<RegionInfo, ServerName> p : currentRegionToServers) {
375      currentRegions.add(p.getFirst());
376    }
377    assertTrue(initialRegions.contains(mergedRegions.getFirst())); //this is the first region
378    assertTrue(initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
379        mergedRegions.getFirst(), 1))); //this is the replica of the first region
380    assertTrue(initialRegions.contains(mergedRegions.getSecond())); //this is the second region
381    assertTrue(initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
382        mergedRegions.getSecond(), 1))); //this is the replica of the second region
383    assertTrue(!initialRegions.contains(currentRegions.get(0))); //this is the new region
384    assertTrue(!initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
385        currentRegions.get(0), 1))); //replica of the new region
386    assertTrue(currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
387        currentRegions.get(0), 1))); //replica of the new region
388    assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
389        mergedRegions.getFirst(), 1))); //replica of the merged region
390    assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
391        mergedRegions.getSecond(), 1))); //replica of the merged region
392  }
393
394  private PairOfSameType<RegionInfo> mergeRegionsAndVerifyRegionNum(
395      HMaster master, TableName tablename,
396      int regionAnum, int regionBnum, int expectedRegionNum) throws Exception {
397    PairOfSameType<RegionInfo> mergedRegions =
398      requestMergeRegion(master, tablename, regionAnum, regionBnum);
399    waitAndVerifyRegionNum(master, tablename, expectedRegionNum);
400    return mergedRegions;
401  }
402
403  private PairOfSameType<RegionInfo> requestMergeRegion(
404      HMaster master, TableName tablename,
405      int regionAnum, int regionBnum) throws Exception {
406    List<Pair<RegionInfo, ServerName>> tableRegions = MetaTableAccessor
407        .getTableRegionsAndLocations(
408            TEST_UTIL.getConnection(), tablename);
409    RegionInfo regionA = tableRegions.get(regionAnum).getFirst();
410    RegionInfo regionB = tableRegions.get(regionBnum).getFirst();
411    ADMIN.mergeRegionsAsync(
412      regionA.getEncodedNameAsBytes(),
413      regionB.getEncodedNameAsBytes(), false);
414    return new PairOfSameType<>(regionA, regionB);
415  }
416
417  private void waitAndVerifyRegionNum(HMaster master, TableName tablename,
418      int expectedRegionNum) throws Exception {
419    List<Pair<RegionInfo, ServerName>> tableRegionsInMeta;
420    List<RegionInfo> tableRegionsInMaster;
421    long timeout = System.currentTimeMillis() + waitTime;
422    while (System.currentTimeMillis() < timeout) {
423      tableRegionsInMeta =
424          MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getConnection(), tablename);
425      tableRegionsInMaster =
426          master.getAssignmentManager().getRegionStates().getRegionsOfTable(tablename);
427      LOG.info(Objects.toString(tableRegionsInMaster));
428      LOG.info(Objects.toString(tableRegionsInMeta));
429      int tableRegionsInMetaSize = tableRegionsInMeta.size();
430      int tableRegionsInMasterSize = tableRegionsInMaster.size();
431      if (tableRegionsInMetaSize == expectedRegionNum
432          && tableRegionsInMasterSize == expectedRegionNum) {
433        break;
434      }
435      Thread.sleep(250);
436    }
437
438    tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(
439        TEST_UTIL.getConnection(), tablename);
440    LOG.info("Regions after merge:" + Joiner.on(',').join(tableRegionsInMeta));
441    assertEquals(expectedRegionNum, tableRegionsInMeta.size());
442  }
443
444  private Table createTableAndLoadData(HMaster master, TableName tablename)
445      throws Exception {
446    return createTableAndLoadData(master, tablename, INITIAL_REGION_NUM, 1);
447  }
448
449  private Table createTableAndLoadData(HMaster master, TableName tablename,
450      int numRegions, int replication) throws Exception {
451    assertTrue("ROWSIZE must > numregions:" + numRegions, ROWSIZE > numRegions);
452    byte[][] splitRows = new byte[numRegions - 1][];
453    for (int i = 0; i < splitRows.length; i++) {
454      splitRows[i] = ROWS[(i + 1) * ROWSIZE / numRegions];
455    }
456
457    Table table = TEST_UTIL.createTable(tablename, FAMILYNAME, splitRows);
458    LOG.info("Created " + table.getName());
459    if (replication > 1) {
460      HBaseTestingUtility.setReplicas(ADMIN, tablename, replication);
461      LOG.info("Set replication of " + replication + " on " + table.getName());
462    }
463    loadData(table);
464    LOG.info("Loaded " + table.getName());
465    verifyRowCount(table, ROWSIZE);
466    LOG.info("Verified " + table.getName());
467
468    List<Pair<RegionInfo, ServerName>> tableRegions;
469    TEST_UTIL.waitUntilAllRegionsAssigned(tablename);
470    LOG.info("All regions assigned for table - " + table.getName());
471    tableRegions = MetaTableAccessor.getTableRegionsAndLocations(
472        TEST_UTIL.getConnection(), tablename);
473    assertEquals("Wrong number of regions in table " + tablename,
474        numRegions * replication, tableRegions.size());
475    LOG.info(tableRegions.size() + "Regions after load: " + Joiner.on(',').join(tableRegions));
476    assertEquals(numRegions * replication, tableRegions.size());
477    return table;
478  }
479
480  private static byte[][] makeN(byte[] base, int n) {
481    byte[][] ret = new byte[n][];
482    for (int i = 0; i < n; i++) {
483      ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%04d", i)));
484    }
485    return ret;
486  }
487
488  private void loadData(Table table) throws IOException {
489    for (int i = 0; i < ROWSIZE; i++) {
490      Put put = new Put(ROWS[i]);
491      put.addColumn(FAMILYNAME, QUALIFIER, Bytes.toBytes(i));
492      table.put(put);
493    }
494  }
495
496  private void verifyRowCount(Table table, int expectedRegionNum)
497      throws IOException {
498    ResultScanner scanner = table.getScanner(new Scan());
499    int rowCount = 0;
500    while (scanner.next() != null) {
501      rowCount++;
502    }
503    assertEquals(expectedRegionNum, rowCount);
504    scanner.close();
505  }
506
507  // Make it public so that JVMClusterUtil can access it.
508  public static class MyMaster extends HMaster {
509    public MyMaster(Configuration conf) throws IOException, KeeperException, InterruptedException {
510      super(conf);
511    }
512
513    @Override
514    protected RSRpcServices createRpcServices() throws IOException {
515      return new MyMasterRpcServices(this);
516    }
517  }
518
519  static class MyMasterRpcServices extends MasterRpcServices {
520    static AtomicBoolean enabled = new AtomicBoolean(false);
521
522    private HMaster myMaster;
523    public MyMasterRpcServices(HMaster master) throws IOException {
524      super(master);
525      myMaster = master;
526    }
527
528    @Override
529    public ReportRegionStateTransitionResponse reportRegionStateTransition(RpcController c,
530        ReportRegionStateTransitionRequest req) throws ServiceException {
531      ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req);
532      if (enabled.get() && req.getTransition(0).getTransitionCode()
533          == TransitionCode.READY_TO_MERGE && !resp.hasErrorMessage()) {
534        RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates();
535        for (RegionState regionState: regionStates.getRegionsStateInTransition()) {
536          // Find the merging_new region and remove it
537          if (regionState.isMergingNew()) {
538            regionStates.deleteRegion(regionState.getRegion());
539          }
540        }
541      }
542      return resp;
543    }
544  }
545}