001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.junit.Assert.fail;
023
024import java.io.IOException;
025import java.util.List;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.Cell;
028import org.apache.hadoop.hbase.CellUtil;
029import org.apache.hadoop.hbase.DoNotRetryIOException;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.HTableDescriptor;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.Waiter;
036import org.apache.hadoop.hbase.client.CompactionState;
037import org.apache.hadoop.hbase.client.Delete;
038import org.apache.hadoop.hbase.client.Get;
039import org.apache.hadoop.hbase.client.Put;
040import org.apache.hadoop.hbase.client.RegionInfo;
041import org.apache.hadoop.hbase.client.Result;
042import org.apache.hadoop.hbase.client.Table;
043import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
044import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
045import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility;
046import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
047import org.apache.hadoop.hbase.procedure2.ProcedureMetrics;
048import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
049import org.apache.hadoop.hbase.regionserver.HRegion;
050import org.apache.hadoop.hbase.testclassification.MasterTests;
051import org.apache.hadoop.hbase.testclassification.MediumTests;
052import org.apache.hadoop.hbase.util.Bytes;
053import org.junit.After;
054import org.junit.AfterClass;
055import org.junit.Before;
056import org.junit.BeforeClass;
057import org.junit.ClassRule;
058import org.junit.Rule;
059import org.junit.Test;
060import org.junit.experimental.categories.Category;
061import org.junit.rules.TestName;
062import org.slf4j.Logger;
063import org.slf4j.LoggerFactory;
064
065@Category({MasterTests.class, MediumTests.class})
066public class TestSplitTableRegionProcedure {
067
068  @ClassRule
069  public static final HBaseClassTestRule CLASS_RULE =
070      HBaseClassTestRule.forClass(TestSplitTableRegionProcedure.class);
071
072  private static final Logger LOG = LoggerFactory.getLogger(TestSplitTableRegionProcedure.class);
073
074  protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
075
076  private static String ColumnFamilyName1 = "cf1";
077  private static String ColumnFamilyName2 = "cf2";
078
079  private static final int startRowNum = 11;
080  private static final int rowCount = 60;
081
082  private AssignmentManager am;
083
084  private ProcedureMetrics splitProcMetrics;
085  private ProcedureMetrics assignProcMetrics;
086  private ProcedureMetrics unassignProcMetrics;
087
088  private long splitSubmittedCount = 0;
089  private long splitFailedCount = 0;
090  private long assignSubmittedCount = 0;
091  private long assignFailedCount = 0;
092  private long unassignSubmittedCount = 0;
093  private long unassignFailedCount = 0;
094
095  @Rule
096  public TestName name = new TestName();
097
098  private static void setupConf(Configuration conf) {
099    conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
100    //testRecoveryAndDoubleExecution requires only one worker
101    conf.setInt(MasterProcedureConstants.MASTER_URGENT_PROCEDURE_THREADS, 0);
102    conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
103  }
104
105  @BeforeClass
106  public static void setupCluster() throws Exception {
107    setupConf(UTIL.getConfiguration());
108    UTIL.startMiniCluster(3);
109  }
110
111  @AfterClass
112  public static void cleanupTest() throws Exception {
113    try {
114      UTIL.shutdownMiniCluster();
115    } catch (Exception e) {
116      LOG.warn("failure shutting down cluster", e);
117    }
118  }
119
120  @Before
121  public void setup() throws Exception {
122    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
123
124    // Turn off balancer so it doesn't cut in and mess up our placements.
125    UTIL.getAdmin().setBalancerRunning(false, true);
126    // Turn off the meta scanner so it don't remove parent on us.
127    UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
128    am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
129    splitProcMetrics = am.getAssignmentManagerMetrics().getSplitProcMetrics();
130    assignProcMetrics = am.getAssignmentManagerMetrics().getAssignProcMetrics();
131    unassignProcMetrics = am.getAssignmentManagerMetrics().getUnassignProcMetrics();
132  }
133
134  @After
135  public void tearDown() throws Exception {
136    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
137    for (HTableDescriptor htd: UTIL.getAdmin().listTables()) {
138      UTIL.deleteTable(htd.getTableName());
139    }
140  }
141
142  @Test
143  public void testSplitTableRegion() throws Exception {
144    final TableName tableName = TableName.valueOf(name.getMethodName());
145    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
146
147    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
148      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
149    insertData(tableName);
150    int splitRowNum = startRowNum + rowCount / 2;
151    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
152
153    assertTrue("not able to find a splittable region", regions != null);
154    assertTrue("not able to find a splittable region", regions.length == 1);
155
156    // collect AM metrics before test
157    collectAssignmentManagerMetrics();
158
159    // Split region of the table
160    long procId = procExec.submitProcedure(
161      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
162    // Wait the completion
163    ProcedureTestingUtility.waitProcedure(procExec, procId);
164    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
165
166    verify(tableName, splitRowNum);
167
168    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
169    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
170    assertEquals(assignSubmittedCount + 2, assignProcMetrics.getSubmittedCounter().getCount());
171    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
172    assertEquals(unassignSubmittedCount + 1, unassignProcMetrics.getSubmittedCounter().getCount());
173    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
174}
175
176  @Test
177  public void testSplitTableRegionNoStoreFile() throws Exception {
178    final TableName tableName = TableName.valueOf(name.getMethodName());
179    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
180
181    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
182      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
183    int splitRowNum = startRowNum + rowCount / 2;
184    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
185
186    assertTrue("not able to find a splittable region", regions != null);
187    assertTrue("not able to find a splittable region", regions.length == 1);
188
189    // collect AM metrics before test
190    collectAssignmentManagerMetrics();
191
192    // Split region of the table
193    long procId = procExec.submitProcedure(
194      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
195    // Wait the completion
196    ProcedureTestingUtility.waitProcedure(procExec, procId);
197    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
198
199    assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 2);
200    assertTrue(UTIL.countRows(tableName) == 0);
201
202    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
203    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
204  }
205
206  @Test
207  public void testSplitTableRegionUnevenDaughter() throws Exception {
208    final TableName tableName = TableName.valueOf(name.getMethodName());
209    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
210
211    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
212      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
213    insertData(tableName);
214    // Split to two daughters with one of them only has 1 row
215    int splitRowNum = startRowNum + rowCount / 4;
216    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
217
218    assertTrue("not able to find a splittable region", regions != null);
219    assertTrue("not able to find a splittable region", regions.length == 1);
220
221    // collect AM metrics before test
222    collectAssignmentManagerMetrics();
223
224    // Split region of the table
225    long procId = procExec.submitProcedure(
226      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
227    // Wait the completion
228    ProcedureTestingUtility.waitProcedure(procExec, procId);
229    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
230
231    verify(tableName, splitRowNum);
232
233    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
234    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
235  }
236
237  @Test
238  public void testSplitTableRegionEmptyDaughter() throws Exception {
239    final TableName tableName = TableName.valueOf(name.getMethodName());
240    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
241
242    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
243      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
244    insertData(tableName);
245    // Split to two daughters with one of them only has 1 row
246    int splitRowNum = startRowNum + rowCount;
247    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
248
249    assertTrue("not able to find a splittable region", regions != null);
250    assertTrue("not able to find a splittable region", regions.length == 1);
251
252    // collect AM metrics before test
253    collectAssignmentManagerMetrics();
254
255    // Split region of the table
256    long procId = procExec.submitProcedure(
257      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
258    // Wait the completion
259    ProcedureTestingUtility.waitProcedure(procExec, procId);
260    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
261
262    // Make sure one daughter has 0 rows.
263    List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
264    assertTrue(daughters.size() == 2);
265    assertTrue(UTIL.countRows(tableName) == rowCount);
266    assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0);
267
268    assertEquals(splitSubmittedCount + 1,
269        splitProcMetrics.getSubmittedCounter().getCount());
270    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
271  }
272
273  @Test
274  public void testSplitTableRegionDeletedRowsDaughter() throws Exception {
275    final TableName tableName = TableName.valueOf(name.getMethodName());
276    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
277
278    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
279      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
280    insertData(tableName);
281    // Split to two daughters with one of them only has 1 row
282    int splitRowNum = rowCount;
283    deleteData(tableName, splitRowNum);
284    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
285
286    assertTrue("not able to find a splittable region", regions != null);
287    assertTrue("not able to find a splittable region", regions.length == 1);
288
289    // collect AM metrics before test
290    collectAssignmentManagerMetrics();
291
292    // Split region of the table
293    long procId = procExec.submitProcedure(
294      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
295    // Wait the completion
296    ProcedureTestingUtility.waitProcedure(procExec, procId);
297    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
298
299    UTIL.getAdmin().majorCompact(tableName);
300    // waiting for the major compaction to complete
301    UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
302      @Override
303      public boolean evaluate() throws IOException {
304        return UTIL.getAdmin().getCompactionState(tableName) == CompactionState.NONE;
305      }
306    });
307
308    // Make sure one daughter has 0 rows.
309    List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
310    assertTrue(daughters.size() == 2);
311    final int currentRowCount = splitRowNum - startRowNum;
312    assertTrue(UTIL.countRows(tableName) == currentRowCount);
313    assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0);
314
315    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
316    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
317  }
318
319  @Test
320  public void testInvalidSplitKey() throws Exception {
321    final TableName tableName = TableName.valueOf(name.getMethodName());
322    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
323
324    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
325      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
326    insertData(tableName);
327
328    assertTrue("not able to find a splittable region", regions != null);
329    assertTrue("not able to find a splittable region", regions.length == 1);
330
331    // collect AM metrics before test
332    collectAssignmentManagerMetrics();
333
334    // Split region of the table with null split key
335    try {
336      long procId1 = procExec.submitProcedure(
337        new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], null));
338      ProcedureTestingUtility.waitProcedure(procExec, procId1);
339      fail("unexpected procedure start with invalid split-key");
340    } catch (DoNotRetryIOException e) {
341      LOG.debug("Expected Split procedure construction failure: " + e.getMessage());
342    }
343
344    assertEquals(splitSubmittedCount, splitProcMetrics.getSubmittedCounter().getCount());
345    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
346  }
347
348  @Test
349  public void testRollbackAndDoubleExecution() throws Exception {
350    final TableName tableName = TableName.valueOf(name.getMethodName());
351    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
352
353    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(
354      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
355    insertData(tableName);
356    int splitRowNum = startRowNum + rowCount / 2;
357    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
358
359    assertTrue("not able to find a splittable region", regions != null);
360    assertTrue("not able to find a splittable region", regions.length == 1);
361    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
362    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
363
364    // collect AM metrics before test
365    collectAssignmentManagerMetrics();
366
367    // Split region of the table
368    long procId = procExec.submitProcedure(
369      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
370
371    // Failing before SPLIT_TABLE_REGION_UPDATE_META we should trigger the rollback
372    // NOTE: the 7 (number of SPLIT_TABLE_REGION_UPDATE_META step) is
373    // hardcoded, so you have to look at this test at least once when you add a new step.
374    int lastStep = 7;
375    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep,
376        true);
377    // check that we have only 1 region
378    assertEquals(1, UTIL.getHBaseAdmin().getTableRegions(tableName).size());
379    UTIL.waitUntilAllRegionsAssigned(tableName);
380    List<HRegion> newRegions = UTIL.getMiniHBaseCluster().getRegions(tableName);
381    assertEquals(1, newRegions.size());
382    verifyData(newRegions.get(0), startRowNum, rowCount,
383    Bytes.toBytes(ColumnFamilyName1), Bytes.toBytes(ColumnFamilyName2));
384
385    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
386    assertEquals(splitFailedCount + 1, splitProcMetrics.getFailedCounter().getCount());
387  }
388
389  @Test
390  public void testRecoveryAndDoubleExecution() throws Exception {
391    final TableName tableName = TableName.valueOf(name.getMethodName());
392    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
393
394    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
395      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
396    insertData(tableName);
397    int splitRowNum = startRowNum + rowCount / 2;
398    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
399
400    assertTrue("not able to find a splittable region", regions != null);
401    assertTrue("not able to find a splittable region", regions.length == 1);
402    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
403    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
404
405    // collect AM metrics before test
406    collectAssignmentManagerMetrics();
407
408    // Split region of the table
409    long procId = procExec.submitProcedure(
410      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
411
412    // Restart the executor and execute the step twice
413    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
414    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
415
416    verify(tableName, splitRowNum);
417
418    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
419    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
420  }
421
422  @Test
423  public void testSplitWithoutPONR() throws Exception {
424    final TableName tableName = TableName.valueOf(name.getMethodName());
425    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
426
427    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
428        procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
429    insertData(tableName);
430    int splitRowNum = startRowNum + rowCount / 2;
431    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
432
433    assertTrue("not able to find a splittable region", regions != null);
434    assertTrue("not able to find a splittable region", regions.length == 1);
435    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
436    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
437
438    // Split region of the table
439    long procId = procExec.submitProcedure(
440        new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
441
442    // Execute until step 7 of split procedure
443    // NOTE: the 7 (number after SPLIT_TABLE_REGION_UPDATE_META step)
444    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, 7, false);
445
446    // Unset Toggle Kill and make ProcExec work correctly
447    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
448    MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
449    ProcedureTestingUtility.waitProcedure(procExec, procId);
450
451    // Even split failed after step 4, it should still works fine
452    verify(tableName, splitRowNum);
453  }
454
455  private void insertData(final TableName tableName) throws IOException, InterruptedException {
456    Table t = UTIL.getConnection().getTable(tableName);
457    Put p;
458    for (int i= 0; i < rowCount / 2; i++) {
459      p = new Put(Bytes.toBytes("" + (startRowNum + i)));
460      p.addColumn(Bytes.toBytes(ColumnFamilyName1), Bytes.toBytes("q1"), Bytes.toBytes(i));
461      p.addColumn(Bytes.toBytes(ColumnFamilyName2), Bytes.toBytes("q2"), Bytes.toBytes(i));
462      t.put(p);
463      p = new Put(Bytes.toBytes("" + (startRowNum + rowCount - i - 1)));
464      p.addColumn(Bytes.toBytes(ColumnFamilyName1), Bytes.toBytes("q1"), Bytes.toBytes(i));
465      p.addColumn(Bytes.toBytes(ColumnFamilyName2), Bytes.toBytes("q2"), Bytes.toBytes(i));
466      t.put(p);
467      if (i % 5 == 0) {
468        UTIL.getAdmin().flush(tableName);
469      }
470    }
471  }
472
473  private void deleteData(
474      final TableName tableName,
475      final int startDeleteRowNum) throws IOException, InterruptedException {
476    Table t = UTIL.getConnection().getTable(tableName);
477    final int numRows = rowCount + startRowNum - startDeleteRowNum;
478    Delete d;
479    for (int i= startDeleteRowNum; i <= numRows + startDeleteRowNum; i++) {
480      d = new Delete(Bytes.toBytes("" + i));
481      t.delete(d);
482      if (i % 5 == 0) {
483        UTIL.getAdmin().flush(tableName);
484      }
485    }
486  }
487
488  private void verify(final TableName tableName, final int splitRowNum) throws IOException {
489    List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
490    assertTrue(daughters.size() == 2);
491    LOG.info("Row Count = " + UTIL.countRows(tableName));
492    assertTrue(UTIL.countRows(tableName) == rowCount);
493    int startRow;
494    int numRows;
495    for (int i = 0; i < daughters.size(); i++) {
496      if (Bytes.compareTo(
497        daughters.get(i).getRegionInfo().getStartKey(), HConstants.EMPTY_BYTE_ARRAY) == 0) {
498        startRow = startRowNum; // first region
499        numRows = splitRowNum - startRowNum;
500      } else {
501        startRow = splitRowNum;
502        numRows = rowCount + startRowNum - splitRowNum;
503      }
504      verifyData(
505        daughters.get(i),
506        startRow,
507        numRows,
508        Bytes.toBytes(ColumnFamilyName1),
509        Bytes.toBytes(ColumnFamilyName2));
510    }
511  }
512
513  private void verifyData(
514      final HRegion newReg,
515      final int startRow,
516      final int numRows,
517      final byte[]... families)
518      throws IOException {
519    for (int i = startRow; i < startRow + numRows; i++) {
520      byte[] row = Bytes.toBytes("" + i);
521      Get get = new Get(row);
522      Result result = newReg.get(get);
523      Cell[] raw = result.rawCells();
524      assertEquals(families.length, result.size());
525      for (int j = 0; j < families.length; j++) {
526        assertTrue(CellUtil.matchingRows(raw[j], row));
527        assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
528      }
529    }
530  }
531
532  private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
533    return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
534  }
535
536  private void collectAssignmentManagerMetrics() {
537    splitSubmittedCount = splitProcMetrics.getSubmittedCounter().getCount();
538    splitFailedCount = splitProcMetrics.getFailedCounter().getCount();
539    assignSubmittedCount = assignProcMetrics.getSubmittedCounter().getCount();
540    assignFailedCount = assignProcMetrics.getFailedCounter().getCount();
541    unassignSubmittedCount = unassignProcMetrics.getSubmittedCounter().getCount();
542    unassignFailedCount = unassignProcMetrics.getFailedCounter().getCount();
543  }
544}