Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.junit.Assert.fail;
023
024import java.io.IOException;
025import java.util.List;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.Cell;
028import org.apache.hadoop.hbase.CellUtil;
029import org.apache.hadoop.hbase.DoNotRetryIOException;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.Waiter;
035import org.apache.hadoop.hbase.client.CompactionState;
036import org.apache.hadoop.hbase.client.Delete;
037import org.apache.hadoop.hbase.client.Get;
038import org.apache.hadoop.hbase.client.Put;
039import org.apache.hadoop.hbase.client.RegionInfo;
040import org.apache.hadoop.hbase.client.Result;
041import org.apache.hadoop.hbase.client.Table;
042import org.apache.hadoop.hbase.client.TableDescriptor;
043import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
044import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
045import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility;
046import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
047import org.apache.hadoop.hbase.procedure2.ProcedureMetrics;
048import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
049import org.apache.hadoop.hbase.regionserver.HRegion;
050import org.apache.hadoop.hbase.testclassification.MasterTests;
051import org.apache.hadoop.hbase.testclassification.MediumTests;
052import org.apache.hadoop.hbase.util.Bytes;
053import org.junit.After;
054import org.junit.AfterClass;
055import org.junit.Before;
056import org.junit.BeforeClass;
057import org.junit.ClassRule;
058import org.junit.Rule;
059import org.junit.Test;
060import org.junit.experimental.categories.Category;
061import org.junit.rules.TestName;
062import org.slf4j.Logger;
063import org.slf4j.LoggerFactory;
064
065@Category({MasterTests.class, MediumTests.class})
066public class TestSplitTableRegionProcedure {
067
068  @ClassRule
069  public static final HBaseClassTestRule CLASS_RULE =
070      HBaseClassTestRule.forClass(TestSplitTableRegionProcedure.class);
071
072  private static final Logger LOG = LoggerFactory.getLogger(TestSplitTableRegionProcedure.class);
073
074  protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
075
076  private static String ColumnFamilyName1 = "cf1";
077  private static String ColumnFamilyName2 = "cf2";
078
079  private static final int startRowNum = 11;
080  private static final int rowCount = 60;
081
082  private AssignmentManager am;
083
084  private ProcedureMetrics splitProcMetrics;
085  private ProcedureMetrics assignProcMetrics;
086  private ProcedureMetrics unassignProcMetrics;
087
088  private long splitSubmittedCount = 0;
089  private long splitFailedCount = 0;
090  private long assignSubmittedCount = 0;
091  private long assignFailedCount = 0;
092  private long unassignSubmittedCount = 0;
093  private long unassignFailedCount = 0;
094
095  @Rule
096  public TestName name = new TestName();
097
098  private static void setupConf(Configuration conf) {
099    conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
100    conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
101  }
102
103  @BeforeClass
104  public static void setupCluster() throws Exception {
105    setupConf(UTIL.getConfiguration());
106    UTIL.startMiniCluster(3);
107  }
108
109  @AfterClass
110  public static void cleanupTest() throws Exception {
111    try {
112      UTIL.shutdownMiniCluster();
113    } catch (Exception e) {
114      LOG.warn("failure shutting down cluster", e);
115    }
116  }
117
118  @Before
119  public void setup() throws Exception {
120    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
121
122    // Turn off balancer so it doesn't cut in and mess up our placements.
123    UTIL.getAdmin().balancerSwitch(false, true);
124    // Turn off the meta scanner so it don't remove parent on us.
125    UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
126    am = UTIL.getHBaseCluster().getMaster().getAssignmentManager();
127    splitProcMetrics = am.getAssignmentManagerMetrics().getSplitProcMetrics();
128    assignProcMetrics = am.getAssignmentManagerMetrics().getAssignProcMetrics();
129    unassignProcMetrics = am.getAssignmentManagerMetrics().getUnassignProcMetrics();
130  }
131
132  @After
133  public void tearDown() throws Exception {
134    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
135    for (TableDescriptor htd : UTIL.getAdmin().listTableDescriptors()) {
136      UTIL.deleteTable(htd.getTableName());
137    }
138  }
139
140  @Test
141  public void testSplitTableRegion() throws Exception {
142    final TableName tableName = TableName.valueOf(name.getMethodName());
143    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
144
145    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
146      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
147    insertData(tableName);
148    int splitRowNum = startRowNum + rowCount / 2;
149    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
150
151    assertTrue("not able to find a splittable region", regions != null);
152    assertTrue("not able to find a splittable region", regions.length == 1);
153
154    // collect AM metrics before test
155    collectAssignmentManagerMetrics();
156
157    // Split region of the table
158    long procId = procExec.submitProcedure(
159      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
160    // Wait the completion
161    ProcedureTestingUtility.waitProcedure(procExec, procId);
162    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
163
164    verify(tableName, splitRowNum);
165
166    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
167    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
168    assertEquals(assignSubmittedCount + 2, assignProcMetrics.getSubmittedCounter().getCount());
169    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
170    assertEquals(unassignSubmittedCount + 1, unassignProcMetrics.getSubmittedCounter().getCount());
171    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
172}
173
174  @Test
175  public void testSplitTableRegionNoStoreFile() throws Exception {
176    final TableName tableName = TableName.valueOf(name.getMethodName());
177    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
178
179    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
180      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
181    int splitRowNum = startRowNum + rowCount / 2;
182    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
183
184    assertTrue("not able to find a splittable region", regions != null);
185    assertTrue("not able to find a splittable region", regions.length == 1);
186
187    // collect AM metrics before test
188    collectAssignmentManagerMetrics();
189
190    // Split region of the table
191    long procId = procExec.submitProcedure(
192      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
193    // Wait the completion
194    ProcedureTestingUtility.waitProcedure(procExec, procId);
195    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
196
197    assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 2);
198    assertTrue(UTIL.countRows(tableName) == 0);
199
200    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
201    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
202  }
203
204  @Test
205  public void testSplitTableRegionUnevenDaughter() throws Exception {
206    final TableName tableName = TableName.valueOf(name.getMethodName());
207    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
208
209    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
210      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
211    insertData(tableName);
212    // Split to two daughters with one of them only has 1 row
213    int splitRowNum = startRowNum + rowCount / 4;
214    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
215
216    assertTrue("not able to find a splittable region", regions != null);
217    assertTrue("not able to find a splittable region", regions.length == 1);
218
219    // collect AM metrics before test
220    collectAssignmentManagerMetrics();
221
222    // Split region of the table
223    long procId = procExec.submitProcedure(
224      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
225    // Wait the completion
226    ProcedureTestingUtility.waitProcedure(procExec, procId);
227    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
228
229    verify(tableName, splitRowNum);
230
231    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
232    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
233  }
234
235  @Test
236  public void testSplitTableRegionEmptyDaughter() throws Exception {
237    final TableName tableName = TableName.valueOf(name.getMethodName());
238    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
239
240    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
241      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
242    insertData(tableName);
243    // Split to two daughters with one of them only has 1 row
244    int splitRowNum = startRowNum + rowCount;
245    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
246
247    assertTrue("not able to find a splittable region", regions != null);
248    assertTrue("not able to find a splittable region", regions.length == 1);
249
250    // collect AM metrics before test
251    collectAssignmentManagerMetrics();
252
253    // Split region of the table
254    long procId = procExec.submitProcedure(
255      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
256    // Wait the completion
257    ProcedureTestingUtility.waitProcedure(procExec, procId);
258    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
259
260    // Make sure one daughter has 0 rows.
261    List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
262    assertTrue(daughters.size() == 2);
263    assertTrue(UTIL.countRows(tableName) == rowCount);
264    assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0);
265
266    assertEquals(splitSubmittedCount + 1,
267        splitProcMetrics.getSubmittedCounter().getCount());
268    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
269  }
270
271  @Test
272  public void testSplitTableRegionDeletedRowsDaughter() throws Exception {
273    final TableName tableName = TableName.valueOf(name.getMethodName());
274    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
275
276    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
277      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
278    insertData(tableName);
279    // Split to two daughters with one of them only has 1 row
280    int splitRowNum = rowCount;
281    deleteData(tableName, splitRowNum);
282    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
283
284    assertTrue("not able to find a splittable region", regions != null);
285    assertTrue("not able to find a splittable region", regions.length == 1);
286
287    // collect AM metrics before test
288    collectAssignmentManagerMetrics();
289
290    // Split region of the table
291    long procId = procExec.submitProcedure(
292      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
293    // Wait the completion
294    ProcedureTestingUtility.waitProcedure(procExec, procId);
295    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
296
297    UTIL.getAdmin().majorCompact(tableName);
298    // waiting for the major compaction to complete
299    UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
300      @Override
301      public boolean evaluate() throws IOException {
302        return UTIL.getAdmin().getCompactionState(tableName) == CompactionState.NONE;
303      }
304    });
305
306    // Make sure one daughter has 0 rows.
307    List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
308    assertTrue(daughters.size() == 2);
309    final int currentRowCount = splitRowNum - startRowNum;
310    assertTrue(UTIL.countRows(tableName) == currentRowCount);
311    assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0);
312
313    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
314    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
315  }
316
317  @Test
318  public void testInvalidSplitKey() throws Exception {
319    final TableName tableName = TableName.valueOf(name.getMethodName());
320    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
321
322    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
323      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
324    insertData(tableName);
325
326    assertTrue("not able to find a splittable region", regions != null);
327    assertTrue("not able to find a splittable region", regions.length == 1);
328
329    // collect AM metrics before test
330    collectAssignmentManagerMetrics();
331
332    // Split region of the table with null split key
333    try {
334      long procId1 = procExec.submitProcedure(
335        new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], null));
336      ProcedureTestingUtility.waitProcedure(procExec, procId1);
337      fail("unexpected procedure start with invalid split-key");
338    } catch (DoNotRetryIOException e) {
339      LOG.debug("Expected Split procedure construction failure: " + e.getMessage());
340    }
341
342    assertEquals(splitSubmittedCount, splitProcMetrics.getSubmittedCounter().getCount());
343    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
344  }
345
346  @Test
347  public void testRollbackAndDoubleExecution() throws Exception {
348    final TableName tableName = TableName.valueOf(name.getMethodName());
349    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
350
351    RegionInfo[] regions = MasterProcedureTestingUtility.createTable(
352      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
353    insertData(tableName);
354    int splitRowNum = startRowNum + rowCount / 2;
355    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
356
357    assertTrue("not able to find a splittable region", regions != null);
358    assertTrue("not able to find a splittable region", regions.length == 1);
359    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
360    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
361
362    // collect AM metrics before test
363    collectAssignmentManagerMetrics();
364
365    // Split region of the table
366    long procId = procExec.submitProcedure(
367      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
368
369    // Failing before SPLIT_TABLE_REGION_UPDATE_META we should trigger the
370    // rollback
371    // NOTE: the 7 (number of SPLIT_TABLE_REGION_UPDATE_META step) is
372    // hardcoded, so you have to look at this test at least once when you add a new step.
373    int lastStep = 7;
374    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep,
375        true);
376    // check that we have only 1 region
377    assertEquals(1, UTIL.getAdmin().getRegions(tableName).size());
378    UTIL.waitUntilAllRegionsAssigned(tableName);
379    List<HRegion> newRegions = UTIL.getMiniHBaseCluster().getRegions(tableName);
380    assertEquals(1, newRegions.size());
381    verifyData(newRegions.get(0), startRowNum, rowCount,
382    Bytes.toBytes(ColumnFamilyName1), Bytes.toBytes(ColumnFamilyName2));
383
384    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
385    assertEquals(splitFailedCount + 1, splitProcMetrics.getFailedCounter().getCount());
386  }
387
388  @Test
389  public void testRecoveryAndDoubleExecution() throws Exception {
390    final TableName tableName = TableName.valueOf(name.getMethodName());
391    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
392
393    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
394      procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
395    insertData(tableName);
396    int splitRowNum = startRowNum + rowCount / 2;
397    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
398
399    assertTrue("not able to find a splittable region", regions != null);
400    assertTrue("not able to find a splittable region", regions.length == 1);
401    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
402    ProcedureTestingUtility.setKillIfHasParent(procExec, false);
403    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
404
405    // collect AM metrics before test
406    collectAssignmentManagerMetrics();
407
408    // Split region of the table
409    long procId = procExec.submitProcedure(
410      new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
411
412    // Restart the executor and execute the step twice
413    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
414    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
415
416    verify(tableName, splitRowNum);
417
418    assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount());
419    assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount());
420  }
421
422  @Test
423  public void testSplitWithoutPONR() throws Exception {
424    final TableName tableName = TableName.valueOf(name.getMethodName());
425    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
426
427    RegionInfo [] regions = MasterProcedureTestingUtility.createTable(
428        procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2);
429    insertData(tableName);
430    int splitRowNum = startRowNum + rowCount / 2;
431    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
432
433    assertTrue("not able to find a splittable region", regions != null);
434    assertTrue("not able to find a splittable region", regions.length == 1);
435    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
436    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
437
438    // Split region of the table
439    long procId = procExec.submitProcedure(
440        new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
441
442    // Execute until step 7 of split procedure
443    // NOTE: the 7 (number after SPLIT_TABLE_REGION_UPDATE_META step)
444    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, 7, false);
445
446    // Unset Toggle Kill and make ProcExec work correctly
447    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
448    MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
449    ProcedureTestingUtility.waitProcedure(procExec, procId);
450
451    // Even split failed after step 4, it should still works fine
452    verify(tableName, splitRowNum);
453  }
454
455  private void insertData(final TableName tableName) throws IOException, InterruptedException {
456    Table t = UTIL.getConnection().getTable(tableName);
457    Put p;
458    for (int i= 0; i < rowCount / 2; i++) {
459      p = new Put(Bytes.toBytes("" + (startRowNum + i)));
460      p.addColumn(Bytes.toBytes(ColumnFamilyName1), Bytes.toBytes("q1"), Bytes.toBytes(i));
461      p.addColumn(Bytes.toBytes(ColumnFamilyName2), Bytes.toBytes("q2"), Bytes.toBytes(i));
462      t.put(p);
463      p = new Put(Bytes.toBytes("" + (startRowNum + rowCount - i - 1)));
464      p.addColumn(Bytes.toBytes(ColumnFamilyName1), Bytes.toBytes("q1"), Bytes.toBytes(i));
465      p.addColumn(Bytes.toBytes(ColumnFamilyName2), Bytes.toBytes("q2"), Bytes.toBytes(i));
466      t.put(p);
467      if (i % 5 == 0) {
468        UTIL.getAdmin().flush(tableName);
469      }
470    }
471  }
472
473  private void deleteData(
474      final TableName tableName,
475      final int startDeleteRowNum) throws IOException, InterruptedException {
476    Table t = UTIL.getConnection().getTable(tableName);
477    final int numRows = rowCount + startRowNum - startDeleteRowNum;
478    Delete d;
479    for (int i= startDeleteRowNum; i <= numRows + startDeleteRowNum; i++) {
480      d = new Delete(Bytes.toBytes("" + i));
481      t.delete(d);
482      if (i % 5 == 0) {
483        UTIL.getAdmin().flush(tableName);
484      }
485    }
486  }
487
488  private void verify(final TableName tableName, final int splitRowNum) throws IOException {
489    List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName);
490    assertTrue(daughters.size() == 2);
491    LOG.info("Row Count = " + UTIL.countRows(tableName));
492    assertTrue(UTIL.countRows(tableName) == rowCount);
493    int startRow;
494    int numRows;
495    for (int i = 0; i < daughters.size(); i++) {
496      if (Bytes.compareTo(
497        daughters.get(i).getRegionInfo().getStartKey(), HConstants.EMPTY_BYTE_ARRAY) == 0) {
498        startRow = startRowNum; // first region
499        numRows = splitRowNum - startRowNum;
500      } else {
501        startRow = splitRowNum;
502        numRows = rowCount + startRowNum - splitRowNum;
503      }
504      verifyData(
505        daughters.get(i),
506        startRow,
507        numRows,
508        Bytes.toBytes(ColumnFamilyName1),
509        Bytes.toBytes(ColumnFamilyName2));
510    }
511  }
512
513  private void verifyData(
514      final HRegion newReg,
515      final int startRow,
516      final int numRows,
517      final byte[]... families)
518      throws IOException {
519    for (int i = startRow; i < startRow + numRows; i++) {
520      byte[] row = Bytes.toBytes("" + i);
521      Get get = new Get(row);
522      Result result = newReg.get(get);
523      Cell[] raw = result.rawCells();
524      assertEquals(families.length, result.size());
525      for (int j = 0; j < families.length; j++) {
526        assertTrue(CellUtil.matchingRows(raw[j], row));
527        assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
528      }
529    }
530  }
531
532  private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
533    return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
534  }
535
536  private void collectAssignmentManagerMetrics() {
537    splitSubmittedCount = splitProcMetrics.getSubmittedCounter().getCount();
538    splitFailedCount = splitProcMetrics.getFailedCounter().getCount();
539    assignSubmittedCount = assignProcMetrics.getSubmittedCounter().getCount();
540    assignFailedCount = assignProcMetrics.getFailedCounter().getCount();
541    unassignSubmittedCount = unassignProcMetrics.getSubmittedCounter().getCount();
542    unassignFailedCount = unassignProcMetrics.getFailedCounter().getCount();
543  }
544}