001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.List;
028import java.util.Objects;
029import java.util.concurrent.TimeUnit;
030import java.util.concurrent.atomic.AtomicBoolean;
031import org.apache.commons.lang3.RandomUtils;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.fs.FileSystem;
034import org.apache.hadoop.fs.Path;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.HBaseTestingUtility;
037import org.apache.hadoop.hbase.MetaTableAccessor;
038import org.apache.hadoop.hbase.MiniHBaseCluster;
039import org.apache.hadoop.hbase.ServerName;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.UnknownRegionException;
042import org.apache.hadoop.hbase.client.Admin;
043import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
044import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
045import org.apache.hadoop.hbase.client.Put;
046import org.apache.hadoop.hbase.client.RegionInfo;
047import org.apache.hadoop.hbase.client.RegionReplicaUtil;
048import org.apache.hadoop.hbase.client.Result;
049import org.apache.hadoop.hbase.client.ResultScanner;
050import org.apache.hadoop.hbase.client.Scan;
051import org.apache.hadoop.hbase.client.Table;
052import org.apache.hadoop.hbase.client.TableDescriptor;
053import org.apache.hadoop.hbase.exceptions.MergeRegionException;
054import org.apache.hadoop.hbase.master.HMaster;
055import org.apache.hadoop.hbase.master.MasterRpcServices;
056import org.apache.hadoop.hbase.master.RegionState;
057import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
058import org.apache.hadoop.hbase.master.assignment.RegionStates;
059import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
060import org.apache.hadoop.hbase.testclassification.LargeTests;
061import org.apache.hadoop.hbase.testclassification.RegionServerTests;
062import org.apache.hadoop.hbase.util.Bytes;
063import org.apache.hadoop.hbase.util.FSUtils;
064import org.apache.hadoop.hbase.util.FutureUtils;
065import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
066import org.apache.hadoop.hbase.util.Pair;
067import org.apache.hadoop.hbase.util.PairOfSameType;
068import org.apache.hadoop.util.StringUtils;
069import org.apache.zookeeper.KeeperException;
070import org.junit.AfterClass;
071import org.junit.BeforeClass;
072import org.junit.ClassRule;
073import org.junit.Rule;
074import org.junit.Test;
075import org.junit.experimental.categories.Category;
076import org.junit.rules.TestName;
077import org.slf4j.Logger;
078import org.slf4j.LoggerFactory;
079
080import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
081import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
082import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
083
084import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
085import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
086import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
087
088@Category({RegionServerTests.class, LargeTests.class})
089public class TestRegionMergeTransactionOnCluster {
090
091  @ClassRule
092  public static final HBaseClassTestRule CLASS_RULE =
093      HBaseClassTestRule.forClass(TestRegionMergeTransactionOnCluster.class);
094
095  private static final Logger LOG =
096      LoggerFactory.getLogger(TestRegionMergeTransactionOnCluster.class);
097
098  @Rule public TestName name = new TestName();
099
100  private static final int NB_SERVERS = 3;
101
102  private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
103  private static final byte[] QUALIFIER = Bytes.toBytes("q");
104
105  private static byte[] ROW = Bytes.toBytes("testRow");
106  private static final int INITIAL_REGION_NUM = 10;
107  private static final int ROWSIZE = 200;
108  private static byte[][] ROWS = makeN(ROW, ROWSIZE);
109
110  private static int waitTime = 60 * 1000;
111
112  static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
113
114  private static HMaster MASTER;
115  private static Admin ADMIN;
116
117  @BeforeClass
118  public static void beforeAllTests() throws Exception {
119    // Start a cluster
120    TEST_UTIL.startMiniCluster(1, NB_SERVERS, null, MyMaster.class, null);
121    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
122    MASTER = cluster.getMaster();
123    MASTER.balanceSwitch(false);
124    ADMIN = TEST_UTIL.getConnection().getAdmin();
125  }
126
127  @AfterClass
128  public static void afterAllTests() throws Exception {
129    TEST_UTIL.shutdownMiniCluster();
130    if (ADMIN != null) ADMIN.close();
131  }
132
133  @Test
134  public void testWholesomeMerge() throws Exception {
135    LOG.info("Starting " + name.getMethodName());
136    final TableName tableName = TableName.valueOf(name.getMethodName());
137
138    // Create table and load data.
139    Table table = createTableAndLoadData(MASTER, tableName);
140    // Merge 1st and 2nd region
141    mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1,
142        INITIAL_REGION_NUM - 1);
143
144    // Merge 2nd and 3th region
145    PairOfSameType<RegionInfo> mergedRegions =
146      mergeRegionsAndVerifyRegionNum(MASTER, tableName, 1, 2,
147        INITIAL_REGION_NUM - 2);
148
149    verifyRowCount(table, ROWSIZE);
150
151    // Randomly choose one of the two merged regions
152    RegionInfo hri = RandomUtils.nextBoolean() ?
153      mergedRegions.getFirst() : mergedRegions.getSecond();
154    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
155    AssignmentManager am = cluster.getMaster().getAssignmentManager();
156    RegionStates regionStates = am.getRegionStates();
157
158    // We should not be able to assign it again
159    am.assign(hri);
160    assertFalse("Merged region can't be assigned",
161      regionStates.isRegionInTransition(hri));
162
163    // We should not be able to unassign it either
164    am.unassign(hri);
165    assertFalse("Merged region can't be unassigned",
166      regionStates.isRegionInTransition(hri));
167
168    table.close();
169  }
170
171  /**
172   * Not really restarting the master. Simulate it by clear of new region
173   * state since it is not persisted, will be lost after master restarts.
174   */
175  @Test
176  public void testMergeAndRestartingMaster() throws Exception {
177    final TableName tableName = TableName.valueOf(name.getMethodName());
178
179    // Create table and load data.
180    Table table = createTableAndLoadData(MASTER, tableName);
181
182    try {
183      MyMasterRpcServices.enabled.set(true);
184
185      // Merge 1st and 2nd region
186      mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1, INITIAL_REGION_NUM - 1);
187    } finally {
188      MyMasterRpcServices.enabled.set(false);
189    }
190
191    table.close();
192  }
193
194  @Test
195  public void testCleanMergeReference() throws Exception {
196    LOG.info("Starting " + name.getMethodName());
197    ADMIN.enableCatalogJanitor(false);
198    try {
199      final TableName tableName = TableName.valueOf(name.getMethodName());
200      // Create table and load data.
201      Table table = createTableAndLoadData(MASTER, tableName);
202      // Merge 1st and 2nd region
203      mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1, INITIAL_REGION_NUM - 1);
204      verifyRowCount(table, ROWSIZE);
205      table.close();
206
207      List<Pair<RegionInfo, ServerName>> tableRegions = MetaTableAccessor
208          .getTableRegionsAndLocations(MASTER.getConnection(), tableName);
209      RegionInfo mergedRegionInfo = tableRegions.get(0).getFirst();
210      TableDescriptor tableDescriptor = MASTER.getTableDescriptors().get(
211          tableName);
212      Result mergedRegionResult = MetaTableAccessor.getRegionResult(
213        MASTER.getConnection(), mergedRegionInfo.getRegionName());
214
215      // contains merge reference in META
216      assertTrue(MetaTableAccessor.hasMergeRegions(mergedRegionResult.rawCells()));
217
218      // merging regions' directory are in the file system all the same
219      List<RegionInfo> p = MetaTableAccessor.getMergeRegions(mergedRegionResult.rawCells());
220      RegionInfo regionA = p.get(0);
221      RegionInfo regionB = p.get(1);
222      FileSystem fs = MASTER.getMasterFileSystem().getFileSystem();
223      Path rootDir = MASTER.getMasterFileSystem().getRootDir();
224
225      Path tabledir = FSUtils.getTableDir(rootDir, mergedRegionInfo.getTable());
226      Path regionAdir = new Path(tabledir, regionA.getEncodedName());
227      Path regionBdir = new Path(tabledir, regionB.getEncodedName());
228      assertTrue(fs.exists(regionAdir));
229      assertTrue(fs.exists(regionBdir));
230
231      ColumnFamilyDescriptor[] columnFamilies = tableDescriptor.getColumnFamilies();
232      HRegionFileSystem hrfs = new HRegionFileSystem(
233        TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo);
234      int count = 0;
235      for(ColumnFamilyDescriptor colFamily : columnFamilies) {
236        count += hrfs.getStoreFiles(colFamily.getName()).size();
237      }
238      ADMIN.compactRegion(mergedRegionInfo.getRegionName());
239      // clean up the merged region store files
240      // wait until merged region have reference file
241      long timeout = System.currentTimeMillis() + waitTime;
242      int newcount = 0;
243      while (System.currentTimeMillis() < timeout) {
244        for(ColumnFamilyDescriptor colFamily : columnFamilies) {
245          newcount += hrfs.getStoreFiles(colFamily.getName()).size();
246        }
247        if(newcount > count) {
248          break;
249        }
250        Thread.sleep(50);
251      }
252      assertTrue(newcount > count);
253      List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
254          .getRegionServerThreads();
255      for (RegionServerThread rs : regionServerThreads) {
256        CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null,
257            rs.getRegionServer(), false);
258        cleaner.chore();
259        Thread.sleep(1000);
260      }
261      while (System.currentTimeMillis() < timeout) {
262        int newcount1 = 0;
263        for(ColumnFamilyDescriptor colFamily : columnFamilies) {
264          newcount1 += hrfs.getStoreFiles(colFamily.getName()).size();
265        }
266        if(newcount1 <= 1) {
267          break;
268        }
269        Thread.sleep(50);
270      }
271      // run CatalogJanitor to clean merge references in hbase:meta and archive the
272      // files of merging regions
273      int cleaned = 0;
274      while (cleaned == 0) {
275        cleaned = ADMIN.runCatalogScan();
276        LOG.debug("catalog janitor returned " + cleaned);
277        Thread.sleep(50);
278        // Cleanup is async so wait till all procedures are done running.
279        ProcedureTestingUtility.waitNoProcedureRunning(
280            TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor());
281      }
282      assertFalse(regionAdir.toString(), fs.exists(regionAdir));
283      assertFalse(regionBdir.toString(), fs.exists(regionBdir));
284      assertTrue(cleaned > 0);
285
286      mergedRegionResult = MetaTableAccessor.getRegionResult(
287        TEST_UTIL.getConnection(), mergedRegionInfo.getRegionName());
288      assertFalse(MetaTableAccessor.hasMergeRegions(mergedRegionResult.rawCells()));
289    } finally {
290      ADMIN.enableCatalogJanitor(true);
291    }
292  }
293
294  /**
295   * This test tests 1, merging region not online;
296   * 2, merging same two regions; 3, merging unknown regions.
297   * They are in one test case so that we don't have to create
298   * many tables, and these tests are simple.
299   */
300  @Test
301  public void testMerge() throws Exception {
302    LOG.info("Starting " + name.getMethodName());
303    final TableName tableName = TableName.valueOf(name.getMethodName());
304    final Admin admin = TEST_UTIL.getAdmin();
305    final int syncWaitTimeout = 10 * 60000; // 10min
306
307    try {
308      // Create table and load data.
309      Table table = createTableAndLoadData(MASTER, tableName);
310      AssignmentManager am = MASTER.getAssignmentManager();
311      List<RegionInfo> regions = am.getRegionStates().getRegionsOfTable(tableName);
312      // Fake offline one region
313      RegionInfo a = regions.get(0);
314      RegionInfo b = regions.get(1);
315      am.unassign(b);
316      am.offlineRegion(b);
317      try {
318        // Merge offline region. Region a is offline here
319        admin.mergeRegionsAsync(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false)
320                .get(syncWaitTimeout, TimeUnit.MILLISECONDS);
321        fail("Offline regions should not be able to merge");
322      } catch (DoNotRetryRegionException ie) {
323        System.out.println(ie);
324        assertTrue(ie instanceof MergeRegionException);
325      }
326
327      try {
328        // Merge the same region: b and b.
329        FutureUtils
330          .get(admin.mergeRegionsAsync(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true));
331        fail("A region should not be able to merge with itself, even forcfully");
332      } catch (IOException ie) {
333        assertTrue("Exception should mention regions not online",
334          StringUtils.stringifyException(ie).contains("region to itself")
335            && ie instanceof MergeRegionException);
336      }
337
338      try {
339        // Merge unknown regions
340        admin.mergeRegionsAsync(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true);
341        fail("Unknown region could not be merged");
342      } catch (IOException ie) {
343        assertTrue("UnknownRegionException should be thrown",
344          ie instanceof UnknownRegionException);
345      }
346      table.close();
347    } finally {
348      TEST_UTIL.deleteTable(tableName);
349    }
350  }
351
352  @Test
353  public void testMergeWithReplicas() throws Exception {
354    final TableName tableName = TableName.valueOf(name.getMethodName());
355    // Create table and load data.
356    createTableAndLoadData(MASTER, tableName, 5, 2);
357    List<Pair<RegionInfo, ServerName>> initialRegionToServers =
358        MetaTableAccessor.getTableRegionsAndLocations(
359            TEST_UTIL.getConnection(), tableName);
360    // Merge 1st and 2nd region
361    PairOfSameType<RegionInfo> mergedRegions = mergeRegionsAndVerifyRegionNum(MASTER, tableName,
362        0, 2, 5 * 2 - 2);
363    List<Pair<RegionInfo, ServerName>> currentRegionToServers =
364        MetaTableAccessor.getTableRegionsAndLocations(
365            TEST_UTIL.getConnection(), tableName);
366    List<RegionInfo> initialRegions = new ArrayList<>();
367    for (Pair<RegionInfo, ServerName> p : initialRegionToServers) {
368      initialRegions.add(p.getFirst());
369    }
370    List<RegionInfo> currentRegions = new ArrayList<>();
371    for (Pair<RegionInfo, ServerName> p : currentRegionToServers) {
372      currentRegions.add(p.getFirst());
373    }
374    assertTrue(initialRegions.contains(mergedRegions.getFirst())); //this is the first region
375    assertTrue(initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
376        mergedRegions.getFirst(), 1))); //this is the replica of the first region
377    assertTrue(initialRegions.contains(mergedRegions.getSecond())); //this is the second region
378    assertTrue(initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
379        mergedRegions.getSecond(), 1))); //this is the replica of the second region
380    assertTrue(!initialRegions.contains(currentRegions.get(0))); //this is the new region
381    assertTrue(!initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
382        currentRegions.get(0), 1))); //replica of the new region
383    assertTrue(currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
384        currentRegions.get(0), 1))); //replica of the new region
385    assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
386        mergedRegions.getFirst(), 1))); //replica of the merged region
387    assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(
388        mergedRegions.getSecond(), 1))); //replica of the merged region
389  }
390
391  private PairOfSameType<RegionInfo> mergeRegionsAndVerifyRegionNum(
392      HMaster master, TableName tablename,
393      int regionAnum, int regionBnum, int expectedRegionNum) throws Exception {
394    PairOfSameType<RegionInfo> mergedRegions =
395      requestMergeRegion(master, tablename, regionAnum, regionBnum);
396    waitAndVerifyRegionNum(master, tablename, expectedRegionNum);
397    return mergedRegions;
398  }
399
400  private PairOfSameType<RegionInfo> requestMergeRegion(
401      HMaster master, TableName tablename,
402      int regionAnum, int regionBnum) throws Exception {
403    List<Pair<RegionInfo, ServerName>> tableRegions = MetaTableAccessor
404        .getTableRegionsAndLocations(
405            TEST_UTIL.getConnection(), tablename);
406    RegionInfo regionA = tableRegions.get(regionAnum).getFirst();
407    RegionInfo regionB = tableRegions.get(regionBnum).getFirst();
408    ADMIN.mergeRegionsAsync(
409      regionA.getEncodedNameAsBytes(),
410      regionB.getEncodedNameAsBytes(), false);
411    return new PairOfSameType<>(regionA, regionB);
412  }
413
414  private void waitAndVerifyRegionNum(HMaster master, TableName tablename,
415      int expectedRegionNum) throws Exception {
416    List<Pair<RegionInfo, ServerName>> tableRegionsInMeta;
417    List<RegionInfo> tableRegionsInMaster;
418    long timeout = System.currentTimeMillis() + waitTime;
419    while (System.currentTimeMillis() < timeout) {
420      tableRegionsInMeta =
421          MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getConnection(), tablename);
422      tableRegionsInMaster =
423          master.getAssignmentManager().getRegionStates().getRegionsOfTable(tablename);
424      LOG.info(Objects.toString(tableRegionsInMaster));
425      LOG.info(Objects.toString(tableRegionsInMeta));
426      int tableRegionsInMetaSize = tableRegionsInMeta.size();
427      int tableRegionsInMasterSize = tableRegionsInMaster.size();
428      if (tableRegionsInMetaSize == expectedRegionNum
429          && tableRegionsInMasterSize == expectedRegionNum) {
430        break;
431      }
432      Thread.sleep(250);
433    }
434
435    tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(
436        TEST_UTIL.getConnection(), tablename);
437    LOG.info("Regions after merge:" + Joiner.on(',').join(tableRegionsInMeta));
438    assertEquals(expectedRegionNum, tableRegionsInMeta.size());
439  }
440
441  private Table createTableAndLoadData(HMaster master, TableName tablename)
442      throws Exception {
443    return createTableAndLoadData(master, tablename, INITIAL_REGION_NUM, 1);
444  }
445
446  private Table createTableAndLoadData(HMaster master, TableName tablename,
447      int numRegions, int replication) throws Exception {
448    assertTrue("ROWSIZE must > numregions:" + numRegions, ROWSIZE > numRegions);
449    byte[][] splitRows = new byte[numRegions - 1][];
450    for (int i = 0; i < splitRows.length; i++) {
451      splitRows[i] = ROWS[(i + 1) * ROWSIZE / numRegions];
452    }
453
454    Table table = TEST_UTIL.createTable(tablename, FAMILYNAME, splitRows);
455    LOG.info("Created " + table.getName());
456    if (replication > 1) {
457      HBaseTestingUtility.setReplicas(ADMIN, tablename, replication);
458      LOG.info("Set replication of " + replication + " on " + table.getName());
459    }
460    loadData(table);
461    LOG.info("Loaded " + table.getName());
462    verifyRowCount(table, ROWSIZE);
463    LOG.info("Verified " + table.getName());
464
465    List<Pair<RegionInfo, ServerName>> tableRegions;
466    TEST_UTIL.waitUntilAllRegionsAssigned(tablename);
467    LOG.info("All regions assigned for table - " + table.getName());
468    tableRegions = MetaTableAccessor.getTableRegionsAndLocations(
469        TEST_UTIL.getConnection(), tablename);
470    assertEquals("Wrong number of regions in table " + tablename,
471        numRegions * replication, tableRegions.size());
472    LOG.info(tableRegions.size() + "Regions after load: " + Joiner.on(',').join(tableRegions));
473    assertEquals(numRegions * replication, tableRegions.size());
474    return table;
475  }
476
477  private static byte[][] makeN(byte[] base, int n) {
478    byte[][] ret = new byte[n][];
479    for (int i = 0; i < n; i++) {
480      ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%04d", i)));
481    }
482    return ret;
483  }
484
485  private void loadData(Table table) throws IOException {
486    for (int i = 0; i < ROWSIZE; i++) {
487      Put put = new Put(ROWS[i]);
488      put.addColumn(FAMILYNAME, QUALIFIER, Bytes.toBytes(i));
489      table.put(put);
490    }
491  }
492
493  private void verifyRowCount(Table table, int expectedRegionNum)
494      throws IOException {
495    ResultScanner scanner = table.getScanner(new Scan());
496    int rowCount = 0;
497    while (scanner.next() != null) {
498      rowCount++;
499    }
500    assertEquals(expectedRegionNum, rowCount);
501    scanner.close();
502  }
503
504  // Make it public so that JVMClusterUtil can access it.
505  public static class MyMaster extends HMaster {
506    public MyMaster(Configuration conf) throws IOException, KeeperException, InterruptedException {
507      super(conf);
508    }
509
510    @Override
511    protected RSRpcServices createRpcServices() throws IOException {
512      return new MyMasterRpcServices(this);
513    }
514  }
515
516  static class MyMasterRpcServices extends MasterRpcServices {
517    static AtomicBoolean enabled = new AtomicBoolean(false);
518
519    private HMaster myMaster;
520    public MyMasterRpcServices(HMaster master) throws IOException {
521      super(master);
522      myMaster = master;
523    }
524
525    @Override
526    public ReportRegionStateTransitionResponse reportRegionStateTransition(RpcController c,
527        ReportRegionStateTransitionRequest req) throws ServiceException {
528      ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(c, req);
529      if (enabled.get() && req.getTransition(0).getTransitionCode()
530          == TransitionCode.READY_TO_MERGE && !resp.hasErrorMessage()) {
531        RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates();
532        for (RegionState regionState: regionStates.getRegionsStateInTransition()) {
533          // Find the merging_new region and remove it
534          if (regionState.isMergingNew()) {
535            regionStates.deleteRegion(regionState.getRegion());
536          }
537        }
538      }
539      return resp;
540    }
541  }
542}