001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.util.ArrayList;
028import java.util.HashMap;
029import java.util.Iterator;
030import java.util.List;
031import java.util.Map;
032import java.util.concurrent.ExecutionException;
033import java.util.concurrent.TimeUnit;
034import java.util.concurrent.atomic.AtomicInteger;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.HBaseTestingUtility;
037import org.apache.hadoop.hbase.HColumnDescriptor;
038import org.apache.hadoop.hbase.HConstants;
039import org.apache.hadoop.hbase.HRegionLocation;
040import org.apache.hadoop.hbase.HTableDescriptor;
041import org.apache.hadoop.hbase.InvalidFamilyOperationException;
042import org.apache.hadoop.hbase.MetaTableAccessor;
043import org.apache.hadoop.hbase.ServerName;
044import org.apache.hadoop.hbase.TableExistsException;
045import org.apache.hadoop.hbase.TableName;
046import org.apache.hadoop.hbase.TableNotDisabledException;
047import org.apache.hadoop.hbase.TableNotEnabledException;
048import org.apache.hadoop.hbase.TableNotFoundException;
049import org.apache.hadoop.hbase.exceptions.MergeRegionException;
050import org.apache.hadoop.hbase.master.LoadBalancer;
051import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
052import org.apache.hadoop.hbase.regionserver.HRegion;
053import org.apache.hadoop.hbase.regionserver.HStore;
054import org.apache.hadoop.hbase.regionserver.HStoreFile;
055import org.apache.hadoop.hbase.testclassification.ClientTests;
056import org.apache.hadoop.hbase.testclassification.LargeTests;
057import org.apache.hadoop.hbase.util.Bytes;
058import org.apache.hadoop.hbase.util.FSUtils;
059import org.apache.hadoop.hbase.util.Pair;
060import org.apache.hadoop.hbase.util.Threads;
061import org.junit.After;
062import org.junit.AfterClass;
063import org.junit.Before;
064import org.junit.BeforeClass;
065import org.junit.ClassRule;
066import org.junit.Rule;
067import org.junit.Test;
068import org.junit.experimental.categories.Category;
069import org.junit.rules.TestName;
070import org.slf4j.Logger;
071import org.slf4j.LoggerFactory;
072
073import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
074import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsRequest;
075
076/**
077 * Class to test HBaseAdmin.
078 * Spins up the minicluster once at test start and then takes it down afterward.
079 * Add any testing of HBaseAdmin functionality here.
080 */
081@Category({LargeTests.class, ClientTests.class})
082public class TestAdmin1 {
083
084  @ClassRule
085  public static final HBaseClassTestRule CLASS_RULE =
086      HBaseClassTestRule.forClass(TestAdmin1.class);
087
088  private static final Logger LOG = LoggerFactory.getLogger(TestAdmin1.class);
089  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
090  private Admin admin;
091
092  @Rule
093  public TestName name = new TestName();
094
095  @BeforeClass
096  public static void setUpBeforeClass() throws Exception {
097    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
098    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
099    TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
100    TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
101    TEST_UTIL.startMiniCluster(3);
102  }
103
104  @AfterClass
105  public static void tearDownAfterClass() throws Exception {
106    TEST_UTIL.shutdownMiniCluster();
107  }
108
109  @Before
110  public void setUp() throws Exception {
111    this.admin = TEST_UTIL.getAdmin();
112  }
113
114  @After
115  public void tearDown() throws Exception {
116    for (HTableDescriptor htd : this.admin.listTables()) {
117      TEST_UTIL.deleteTable(htd.getTableName());
118    }
119  }
120
121  @Test
122  public void testSplitFlushCompactUnknownTable() throws InterruptedException {
123    final TableName unknowntable = TableName.valueOf(name.getMethodName());
124    Exception exception = null;
125    try {
126      this.admin.compact(unknowntable);
127    } catch (IOException e) {
128      exception = e;
129    }
130    assertTrue(exception instanceof TableNotFoundException);
131
132    exception = null;
133    try {
134      this.admin.flush(unknowntable);
135    } catch (IOException e) {
136      exception = e;
137    }
138    assertTrue(exception instanceof TableNotFoundException);
139
140    exception = null;
141    try {
142      this.admin.split(unknowntable);
143    } catch (IOException e) {
144      exception = e;
145    }
146    assertTrue(exception instanceof TableNotFoundException);
147  }
148
149  @Test
150  public void testDeleteEditUnknownColumnFamilyAndOrTable() throws IOException {
151    // Test we get exception if we try to
152    final TableName nonexistentTable = TableName.valueOf("nonexistent");
153    final byte[] nonexistentColumn = Bytes.toBytes("nonexistent");
154    HColumnDescriptor nonexistentHcd = new HColumnDescriptor(nonexistentColumn);
155    Exception exception = null;
156    try {
157      this.admin.addColumnFamily(nonexistentTable, nonexistentHcd);
158    } catch (IOException e) {
159      exception = e;
160    }
161    assertTrue(exception instanceof TableNotFoundException);
162
163    exception = null;
164    try {
165      this.admin.deleteTable(nonexistentTable);
166    } catch (IOException e) {
167      exception = e;
168    }
169    assertTrue(exception instanceof TableNotFoundException);
170
171    exception = null;
172    try {
173      this.admin.deleteColumnFamily(nonexistentTable, nonexistentColumn);
174    } catch (IOException e) {
175      exception = e;
176    }
177    assertTrue(exception instanceof TableNotFoundException);
178
179    exception = null;
180    try {
181      this.admin.disableTable(nonexistentTable);
182    } catch (IOException e) {
183      exception = e;
184    }
185    assertTrue(exception instanceof TableNotFoundException);
186
187    exception = null;
188    try {
189      this.admin.enableTable(nonexistentTable);
190    } catch (IOException e) {
191      exception = e;
192    }
193    assertTrue(exception instanceof TableNotFoundException);
194
195    exception = null;
196    try {
197      this.admin.modifyColumnFamily(nonexistentTable, nonexistentHcd);
198    } catch (IOException e) {
199      exception = e;
200    }
201    assertTrue(exception instanceof TableNotFoundException);
202
203    exception = null;
204    try {
205      HTableDescriptor htd = new HTableDescriptor(nonexistentTable);
206      htd.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
207      this.admin.modifyTable(htd.getTableName(), htd);
208    } catch (IOException e) {
209      exception = e;
210    }
211    assertTrue(exception instanceof TableNotFoundException);
212
213    // Now make it so at least the table exists and then do tests against a
214    // nonexistent column family -- see if we get right exceptions.
215    final TableName tableName = TableName.valueOf(name.getMethodName() + System.currentTimeMillis());
216    HTableDescriptor htd = new HTableDescriptor(tableName);
217    htd.addFamily(new HColumnDescriptor("cf"));
218    this.admin.createTable(htd);
219    try {
220      exception = null;
221      try {
222        this.admin.deleteColumnFamily(htd.getTableName(), nonexistentHcd.getName());
223      } catch (IOException e) {
224        exception = e;
225      }
226      assertTrue("found=" + exception.getClass().getName(),
227          exception instanceof InvalidFamilyOperationException);
228
229      exception = null;
230      try {
231        this.admin.modifyColumnFamily(htd.getTableName(), nonexistentHcd);
232      } catch (IOException e) {
233        exception = e;
234      }
235      assertTrue("found=" + exception.getClass().getName(),
236          exception instanceof InvalidFamilyOperationException);
237    } finally {
238      this.admin.disableTable(tableName);
239      this.admin.deleteTable(tableName);
240    }
241  }
242
243  @Test
244  public void testDisableAndEnableTable() throws IOException {
245    final byte [] row = Bytes.toBytes("row");
246    final byte [] qualifier = Bytes.toBytes("qualifier");
247    final byte [] value = Bytes.toBytes("value");
248    final TableName table = TableName.valueOf(name.getMethodName());
249    Table ht = TEST_UTIL.createTable(table, HConstants.CATALOG_FAMILY);
250    Put put = new Put(row);
251    put.addColumn(HConstants.CATALOG_FAMILY, qualifier, value);
252    ht.put(put);
253    Get get = new Get(row);
254    get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
255    ht.get(get);
256
257    this.admin.disableTable(ht.getName());
258    assertTrue("Table must be disabled.", TEST_UTIL.getHBaseCluster()
259        .getMaster().getTableStateManager().isTableState(
260            ht.getName(), TableState.State.DISABLED));
261    assertEquals(TableState.State.DISABLED, getStateFromMeta(table));
262
263    // Test that table is disabled
264    get = new Get(row);
265    get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
266    boolean ok = false;
267    try {
268      ht.get(get);
269    } catch (TableNotEnabledException e) {
270      ok = true;
271    }
272    ok = false;
273    // verify that scan encounters correct exception
274    Scan scan = new Scan();
275    try {
276      ResultScanner scanner = ht.getScanner(scan);
277      Result res = null;
278      do {
279        res = scanner.next();
280      } while (res != null);
281    } catch (TableNotEnabledException e) {
282      ok = true;
283    }
284    assertTrue(ok);
285    this.admin.enableTable(table);
286    assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster()
287        .getMaster().getTableStateManager().isTableState(
288            ht.getName(), TableState.State.ENABLED));
289    assertEquals(TableState.State.ENABLED, getStateFromMeta(table));
290
291    // Test that table is enabled
292    try {
293      ht.get(get);
294    } catch (RetriesExhaustedException e) {
295      ok = false;
296    }
297    assertTrue(ok);
298    ht.close();
299  }
300
301  private TableState.State getStateFromMeta(TableName table) throws IOException {
302    TableState state =
303        MetaTableAccessor.getTableState(TEST_UTIL.getConnection(), table);
304    assertNotNull(state);
305    return state.getState();
306  }
307
308  @Test
309  public void testDisableAndEnableTables() throws IOException {
310    final byte [] row = Bytes.toBytes("row");
311    final byte [] qualifier = Bytes.toBytes("qualifier");
312    final byte [] value = Bytes.toBytes("value");
313    final TableName table1 = TableName.valueOf(name.getMethodName() + "1");
314    final TableName table2 = TableName.valueOf(name.getMethodName() + "2");
315    Table ht1 = TEST_UTIL.createTable(table1, HConstants.CATALOG_FAMILY);
316    Table ht2 = TEST_UTIL.createTable(table2, HConstants.CATALOG_FAMILY);
317    Put put = new Put(row);
318    put.addColumn(HConstants.CATALOG_FAMILY, qualifier, value);
319    ht1.put(put);
320    ht2.put(put);
321    Get get = new Get(row);
322    get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
323    ht1.get(get);
324    ht2.get(get);
325
326    this.admin.disableTables("testDisableAndEnableTable.*");
327
328    // Test that tables are disabled
329    get = new Get(row);
330    get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
331    boolean ok = false;
332    try {
333      ht1.get(get);
334      ht2.get(get);
335    } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
336      ok = true;
337    }
338
339    assertEquals(TableState.State.DISABLED, getStateFromMeta(table1));
340    assertEquals(TableState.State.DISABLED, getStateFromMeta(table2));
341
342
343    assertTrue(ok);
344    this.admin.enableTables("testDisableAndEnableTable.*");
345
346    // Test that tables are enabled
347    try {
348      ht1.get(get);
349    } catch (IOException e) {
350      ok = false;
351    }
352    try {
353      ht2.get(get);
354    } catch (IOException e) {
355      ok = false;
356    }
357    assertTrue(ok);
358
359    ht1.close();
360    ht2.close();
361
362    assertEquals(TableState.State.ENABLED, getStateFromMeta(table1));
363    assertEquals(TableState.State.ENABLED, getStateFromMeta(table2));
364  }
365
366  @Test
367  public void testCreateTable() throws IOException {
368    HTableDescriptor [] tables = admin.listTables();
369    int numTables = tables.length;
370    final TableName tableName = TableName.valueOf(name.getMethodName());
371    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
372    tables = this.admin.listTables();
373    assertEquals(numTables + 1, tables.length);
374    assertTrue("Table must be enabled.",
375        TEST_UTIL.getHBaseCluster().getMaster().getTableStateManager()
376            .isTableState(tableName, TableState.State.ENABLED));
377    assertEquals(TableState.State.ENABLED, getStateFromMeta(tableName));
378  }
379
380  @Test
381  public void testTruncateTable() throws IOException {
382    testTruncateTable(TableName.valueOf(name.getMethodName()), false);
383  }
384
385  @Test
386  public void testTruncateTablePreservingSplits() throws IOException {
387    testTruncateTable(TableName.valueOf(name.getMethodName()), true);
388  }
389
390  private void testTruncateTable(final TableName tableName, boolean preserveSplits)
391      throws IOException {
392    byte[][] splitKeys = new byte[2][];
393    splitKeys[0] = Bytes.toBytes(4);
394    splitKeys[1] = Bytes.toBytes(8);
395
396    // Create & Fill the table
397    Table table = TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY, splitKeys);
398    try {
399      TEST_UTIL.loadNumericRows(table, HConstants.CATALOG_FAMILY, 0, 10);
400      assertEquals(10, TEST_UTIL.countRows(table));
401    } finally {
402      table.close();
403    }
404    assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
405
406    // Truncate & Verify
407    this.admin.disableTable(tableName);
408    this.admin.truncateTable(tableName, preserveSplits);
409    table = TEST_UTIL.getConnection().getTable(tableName);
410    try {
411      assertEquals(0, TEST_UTIL.countRows(table));
412    } finally {
413      table.close();
414    }
415    if (preserveSplits) {
416      assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
417    } else {
418      assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
419    }
420  }
421
422  @Test
423  public void testGetTableDescriptor() throws IOException {
424    HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
425    HColumnDescriptor fam2 = new HColumnDescriptor("fam2");
426    HColumnDescriptor fam3 = new HColumnDescriptor("fam3");
427    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
428    htd.addFamily(fam1);
429    htd.addFamily(fam2);
430    htd.addFamily(fam3);
431    this.admin.createTable(htd);
432    Table table = TEST_UTIL.getConnection().getTable(htd.getTableName());
433    TableDescriptor confirmedHtd = table.getDescriptor();
434    assertEquals(0, TableDescriptor.COMPARATOR.compare(htd, confirmedHtd));
435    MetaTableAccessor.fullScanMetaAndPrint(TEST_UTIL.getConnection());
436    table.close();
437  }
438
439  @Test
440  public void testCompactionTimestamps() throws Exception {
441    HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
442    final TableName tableName = TableName.valueOf(name.getMethodName());
443    HTableDescriptor htd = new HTableDescriptor(tableName);
444    htd.addFamily(fam1);
445    this.admin.createTable(htd);
446    Table table = TEST_UTIL.getConnection().getTable(htd.getTableName());
447    long ts = this.admin.getLastMajorCompactionTimestamp(tableName);
448    assertEquals(0, ts);
449    Put p = new Put(Bytes.toBytes("row1"));
450    p.addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("fam1"), Bytes.toBytes("fam1"));
451    table.put(p);
452    ts = this.admin.getLastMajorCompactionTimestamp(tableName);
453    // no files written -> no data
454    assertEquals(0, ts);
455
456    this.admin.flush(tableName);
457    ts = this.admin.getLastMajorCompactionTimestamp(tableName);
458    // still 0, we flushed a file, but no major compaction happened
459    assertEquals(0, ts);
460
461    byte[] regionName;
462    try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
463      regionName = l.getAllRegionLocations().get(0).getRegionInfo().getRegionName();
464    }
465    long ts1 = this.admin.getLastMajorCompactionTimestampForRegion(regionName);
466    assertEquals(ts, ts1);
467    p = new Put(Bytes.toBytes("row2"));
468    p.addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("fam1"), Bytes.toBytes("fam1"));
469    table.put(p);
470    this.admin.flush(tableName);
471    ts = this.admin.getLastMajorCompactionTimestamp(tableName);
472    // make sure the region API returns the same value, as the old file is still around
473    assertEquals(ts1, ts);
474
475    TEST_UTIL.compact(tableName, true);
476    table.put(p);
477    // forces a wait for the compaction
478    this.admin.flush(tableName);
479    ts = this.admin.getLastMajorCompactionTimestamp(tableName);
480    // after a compaction our earliest timestamp will have progressed forward
481    assertTrue(ts > ts1);
482
483    // region api still the same
484    ts1 = this.admin.getLastMajorCompactionTimestampForRegion(regionName);
485    assertEquals(ts, ts1);
486    table.put(p);
487    this.admin.flush(tableName);
488    ts = this.admin.getLastMajorCompactionTimestamp(tableName);
489    assertEquals(ts, ts1);
490    table.close();
491  }
492
493  @Test
494  public void testHColumnValidName() {
495       boolean exceptionThrown;
496       try {
497         new HColumnDescriptor("\\test\\abc");
498       } catch(IllegalArgumentException iae) {
499           exceptionThrown = true;
500           assertTrue(exceptionThrown);
501       }
502   }
503
504  /**
505   * Verify schema change for read only table
506   */
507  @Test
508  public void testReadOnlyTableModify() throws IOException, InterruptedException {
509    final TableName tableName = TableName.valueOf(name.getMethodName());
510    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
511
512    // Make table read only
513    TableDescriptor htd = TableDescriptorBuilder.newBuilder(this.admin.getDescriptor(tableName))
514      .setReadOnly(true).build();
515    admin.modifyTable(htd);
516
517    // try to modify the read only table now
518    htd = TableDescriptorBuilder.newBuilder(this.admin.getDescriptor(tableName))
519      .setCompactionEnabled(false).build();
520    admin.modifyTable(htd);
521    // Delete the table
522    this.admin.disableTable(tableName);
523    this.admin.deleteTable(tableName);
524    assertFalse(this.admin.tableExists(tableName));
525  }
526
527  @Test(expected = TableNotDisabledException.class)
528  public void testModifyRegionReplicasEnabledTable() throws Exception {
529    final TableName tableName = TableName.valueOf(name.getMethodName());
530    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
531
532    // Modify region replication count
533    TableDescriptor htd = TableDescriptorBuilder.newBuilder(admin.getDescriptor(tableName))
534        .setRegionReplication(3).build();
535    try {
536      // try to modify the region replication count without disabling the table
537      admin.modifyTable(htd);
538      fail("Expected an exception");
539    } finally {
540      // Delete the table
541      admin.disableTable(tableName);
542      admin.deleteTable(tableName);
543      assertFalse(admin.tableExists(tableName));
544    }
545  }
546
547  /**
548   * Verify schema modification takes.
549   */
550  @Test
551  public void testOnlineChangeTableSchema() throws IOException, InterruptedException {
552    final TableName tableName = TableName.valueOf(name.getMethodName());
553    HTableDescriptor [] tables = admin.listTables();
554    int numTables = tables.length;
555    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
556    tables = this.admin.listTables();
557    assertEquals(numTables + 1, tables.length);
558
559    // FIRST, do htabledescriptor changes.
560    HTableDescriptor htd = this.admin.getTableDescriptor(tableName);
561    // Make a copy and assert copy is good.
562    HTableDescriptor copy = new HTableDescriptor(htd);
563    assertTrue(htd.equals(copy));
564    // Now amend the copy. Introduce differences.
565    long newFlushSize = htd.getMemStoreFlushSize() / 2;
566    if (newFlushSize <=0) {
567      newFlushSize = HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE / 2;
568    }
569    copy.setMemStoreFlushSize(newFlushSize);
570    final String key = "anyoldkey";
571    assertTrue(htd.getValue(key) == null);
572    copy.setValue(key, key);
573    boolean expectedException = false;
574    try {
575      admin.modifyTable(tableName, copy);
576    } catch (TableNotDisabledException re) {
577      expectedException = true;
578    }
579    assertFalse(expectedException);
580    HTableDescriptor modifiedHtd = new HTableDescriptor(this.admin.getTableDescriptor(tableName));
581    assertFalse(htd.equals(modifiedHtd));
582    assertTrue(copy.equals(modifiedHtd));
583    assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize());
584    assertEquals(key, modifiedHtd.getValue(key));
585
586    // Now work on column family changes.
587    int countOfFamilies = modifiedHtd.getFamilies().size();
588    assertTrue(countOfFamilies > 0);
589    HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next();
590    int maxversions = hcd.getMaxVersions();
591    final int newMaxVersions = maxversions + 1;
592    hcd.setMaxVersions(newMaxVersions);
593    final byte [] hcdName = hcd.getName();
594    expectedException = false;
595    try {
596      this.admin.modifyColumnFamily(tableName, hcd);
597    } catch (TableNotDisabledException re) {
598      expectedException = true;
599    }
600    assertFalse(expectedException);
601    modifiedHtd = this.admin.getTableDescriptor(tableName);
602    HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName);
603    assertEquals(newMaxVersions, modifiedHcd.getMaxVersions());
604
605    // Try adding a column
606    assertFalse(this.admin.isTableDisabled(tableName));
607    final String xtracolName = "xtracol";
608    HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName);
609    xtracol.setValue(xtracolName, xtracolName);
610    expectedException = false;
611    try {
612      this.admin.addColumnFamily(tableName, xtracol);
613    } catch (TableNotDisabledException re) {
614      expectedException = true;
615    }
616    // Add column should work even if the table is enabled
617    assertFalse(expectedException);
618    modifiedHtd = this.admin.getTableDescriptor(tableName);
619    hcd = modifiedHtd.getFamily(xtracol.getName());
620    assertTrue(hcd != null);
621    assertTrue(hcd.getValue(xtracolName).equals(xtracolName));
622
623    // Delete the just-added column.
624    this.admin.deleteColumnFamily(tableName, xtracol.getName());
625    modifiedHtd = this.admin.getTableDescriptor(tableName);
626    hcd = modifiedHtd.getFamily(xtracol.getName());
627    assertTrue(hcd == null);
628
629    // Delete the table
630    this.admin.disableTable(tableName);
631    this.admin.deleteTable(tableName);
632    this.admin.listTables();
633    assertFalse(this.admin.tableExists(tableName));
634  }
635
636  private void verifyRoundRobinDistribution(RegionLocator regionLocator, int expectedRegions)
637      throws IOException {
638    int numRS = TEST_UTIL.getMiniHBaseCluster().getNumLiveRegionServers();
639    List<HRegionLocation> regions = regionLocator.getAllRegionLocations();
640    Map<ServerName, List<RegionInfo>> server2Regions = new HashMap<>();
641    for (HRegionLocation loc : regions) {
642      ServerName server = loc.getServerName();
643      List<RegionInfo> regs = server2Regions.get(server);
644      if (regs == null) {
645        regs = new ArrayList<>();
646        server2Regions.put(server, regs);
647      }
648      regs.add(loc.getRegionInfo());
649    }
650    boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
651    if (tablesOnMaster) {
652      // Ignore the master region server,
653      // which contains less regions by intention.
654      numRS--;
655    }
656    float average = (float) expectedRegions/numRS;
657    int min = (int)Math.floor(average);
658    int max = (int)Math.ceil(average);
659    for (List<RegionInfo> regionList : server2Regions.values()) {
660      assertTrue("numRS=" + numRS + ", min=" + min + ", max=" + max +
661        ", size=" + regionList.size() + ", tablesOnMaster=" + tablesOnMaster,
662      regionList.size() == min || regionList.size() == max);
663    }
664  }
665
666  @Test
667  public void testCreateTableNumberOfRegions() throws IOException, InterruptedException {
668    final TableName tableName = TableName.valueOf(name.getMethodName());
669    HTableDescriptor desc = new HTableDescriptor(tableName);
670    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
671    admin.createTable(desc);
672    List<HRegionLocation> regions;
673    try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
674      regions = l.getAllRegionLocations();
675      assertEquals("Table should have only 1 region", 1, regions.size());
676    }
677
678    TableName TABLE_2 = TableName.valueOf(tableName.getNameAsString() + "_2");
679    desc = new HTableDescriptor(TABLE_2);
680    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
681    admin.createTable(desc, new byte[][]{new byte[]{42}});
682    try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(TABLE_2)) {
683      regions = l.getAllRegionLocations();
684      assertEquals("Table should have only 2 region", 2, regions.size());
685    }
686
687    TableName TABLE_3 = TableName.valueOf(tableName.getNameAsString() + "_3");
688    desc = new HTableDescriptor(TABLE_3);
689    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
690    admin.createTable(desc, "a".getBytes(), "z".getBytes(), 3);
691    try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(TABLE_3)) {
692      regions = l.getAllRegionLocations();
693      assertEquals("Table should have only 3 region", 3, regions.size());
694    }
695
696    TableName TABLE_4 = TableName.valueOf(tableName.getNameAsString() + "_4");
697    desc = new HTableDescriptor(TABLE_4);
698    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
699    try {
700      admin.createTable(desc, "a".getBytes(), "z".getBytes(), 2);
701      fail("Should not be able to create a table with only 2 regions using this API.");
702    } catch (IllegalArgumentException eae) {
703    // Expected
704    }
705
706    TableName TABLE_5 = TableName.valueOf(tableName.getNameAsString() + "_5");
707    desc = new HTableDescriptor(TABLE_5);
708    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
709    admin.createTable(desc, new byte[] { 1 }, new byte[] { 127 }, 16);
710    try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(TABLE_5)) {
711      regions = l.getAllRegionLocations();
712      assertEquals("Table should have 16 region", 16, regions.size());
713    }
714  }
715
716  @Test
717  public void testCreateTableWithRegions() throws IOException, InterruptedException {
718    final TableName tableName = TableName.valueOf(name.getMethodName());
719
720    byte [][] splitKeys = {
721        new byte [] { 1, 1, 1 },
722        new byte [] { 2, 2, 2 },
723        new byte [] { 3, 3, 3 },
724        new byte [] { 4, 4, 4 },
725        new byte [] { 5, 5, 5 },
726        new byte [] { 6, 6, 6 },
727        new byte [] { 7, 7, 7 },
728        new byte [] { 8, 8, 8 },
729        new byte [] { 9, 9, 9 },
730    };
731    int expectedRegions = splitKeys.length + 1;
732
733    HTableDescriptor desc = new HTableDescriptor(tableName);
734    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
735    admin.createTable(desc, splitKeys);
736
737    boolean tableAvailable = admin.isTableAvailable(tableName, splitKeys);
738    assertTrue("Table should be created with splitKyes + 1 rows in META", tableAvailable);
739
740    List<HRegionLocation> regions;
741    Iterator<HRegionLocation> hris;
742    RegionInfo hri;
743    ClusterConnection conn = (ClusterConnection) TEST_UTIL.getConnection();
744    try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
745      regions = l.getAllRegionLocations();
746
747      assertEquals("Tried to create " + expectedRegions + " regions " +
748              "but only found " + regions.size(), expectedRegions, regions.size());
749      System.err.println("Found " + regions.size() + " regions");
750
751      hris = regions.iterator();
752      hri = hris.next().getRegionInfo();
753      assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
754      assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[0]));
755      hri = hris.next().getRegionInfo();
756      assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[0]));
757      assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[1]));
758      hri = hris.next().getRegionInfo();
759      assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[1]));
760      assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[2]));
761      hri = hris.next().getRegionInfo();
762      assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[2]));
763      assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[3]));
764      hri = hris.next().getRegionInfo();
765      assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[3]));
766      assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[4]));
767      hri = hris.next().getRegionInfo();
768      assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[4]));
769      assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[5]));
770      hri = hris.next().getRegionInfo();
771      assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[5]));
772      assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[6]));
773      hri = hris.next().getRegionInfo();
774      assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[6]));
775      assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[7]));
776      hri = hris.next().getRegionInfo();
777      assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[7]));
778      assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[8]));
779      hri = hris.next().getRegionInfo();
780      assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8]));
781      assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
782
783      verifyRoundRobinDistribution(l, expectedRegions);
784    }
785
786
787
788    // Now test using start/end with a number of regions
789
790    // Use 80 bit numbers to make sure we aren't limited
791    byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
792    byte [] endKey =   { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
793
794    // Splitting into 10 regions, we expect (null,1) ... (9, null)
795    // with (1,2) (2,3) (3,4) (4,5) (5,6) (6,7) (7,8) (8,9) in the middle
796
797    expectedRegions = 10;
798
799    TableName TABLE_2 = TableName.valueOf(tableName.getNameAsString() + "_2");
800
801    desc = new HTableDescriptor(TABLE_2);
802    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
803    admin = TEST_UTIL.getAdmin();
804    admin.createTable(desc, startKey, endKey, expectedRegions);
805
806    try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(TABLE_2)) {
807      regions = l.getAllRegionLocations();
808      assertEquals("Tried to create " + expectedRegions + " regions " +
809          "but only found " + regions.size(), expectedRegions, regions.size());
810      System.err.println("Found " + regions.size() + " regions");
811
812      hris = regions.iterator();
813      hri = hris.next().getRegionInfo();
814      assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
815      assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }));
816      hri = hris.next().getRegionInfo();
817      assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }));
818      assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }));
819      hri = hris.next().getRegionInfo();
820      assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }));
821      assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }));
822      hri = hris.next().getRegionInfo();
823      assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }));
824      assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }));
825      hri = hris.next().getRegionInfo();
826      assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }));
827      assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }));
828      hri = hris.next().getRegionInfo();
829      assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }));
830      assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 }));
831      hri = hris.next().getRegionInfo();
832      assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 }));
833      assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }));
834      hri = hris.next().getRegionInfo();
835      assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }));
836      assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 }));
837      hri = hris.next().getRegionInfo();
838      assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 }));
839      assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }));
840      hri = hris.next().getRegionInfo();
841      assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }));
842      assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
843
844      verifyRoundRobinDistribution(l, expectedRegions);
845    }
846
847    // Try once more with something that divides into something infinite
848
849    startKey = new byte [] { 0, 0, 0, 0, 0, 0 };
850    endKey = new byte [] { 1, 0, 0, 0, 0, 0 };
851
852    expectedRegions = 5;
853
854    TableName TABLE_3 = TableName.valueOf(tableName.getNameAsString() + "_3");
855
856    desc = new HTableDescriptor(TABLE_3);
857    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
858    admin = TEST_UTIL.getAdmin();
859    admin.createTable(desc, startKey, endKey, expectedRegions);
860
861
862    try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(TABLE_3)) {
863      regions = l.getAllRegionLocations();
864      assertEquals("Tried to create " + expectedRegions + " regions " +
865          "but only found " + regions.size(), expectedRegions, regions.size());
866      System.err.println("Found " + regions.size() + " regions");
867
868      verifyRoundRobinDistribution(l, expectedRegions);
869    }
870
871
872    // Try an invalid case where there are duplicate split keys
873    splitKeys = new byte [][] {
874        new byte [] { 1, 1, 1 },
875        new byte [] { 2, 2, 2 },
876        new byte [] { 3, 3, 3 },
877        new byte [] { 2, 2, 2 }
878    };
879
880    TableName TABLE_4 = TableName.valueOf(tableName.getNameAsString() + "_4");
881    desc = new HTableDescriptor(TABLE_4);
882    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
883    try {
884      admin.createTable(desc, splitKeys);
885      assertTrue("Should not be able to create this table because of " +
886          "duplicate split keys", false);
887    } catch(IllegalArgumentException iae) {
888      // Expected
889    }
890  }
891
892  @Test
893  public void testTableAvailableWithRandomSplitKeys() throws Exception {
894    final TableName tableName = TableName.valueOf(name.getMethodName());
895    HTableDescriptor desc = new HTableDescriptor(tableName);
896    desc.addFamily(new HColumnDescriptor("col"));
897    byte[][] splitKeys = new byte[1][];
898    splitKeys = new byte [][] {
899        new byte [] { 1, 1, 1 },
900        new byte [] { 2, 2, 2 }
901    };
902    admin.createTable(desc);
903    boolean tableAvailable = admin.isTableAvailable(tableName, splitKeys);
904    assertFalse("Table should be created with 1 row in META", tableAvailable);
905  }
906
907  @Test
908  public void testCreateTableWithOnlyEmptyStartRow() throws IOException {
909    final byte[] tableName = Bytes.toBytes(name.getMethodName());
910    byte[][] splitKeys = new byte[1][];
911    splitKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
912    HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
913    desc.addFamily(new HColumnDescriptor("col"));
914    try {
915      admin.createTable(desc, splitKeys);
916      fail("Test case should fail as empty split key is passed.");
917    } catch (IllegalArgumentException e) {
918    }
919  }
920
921  @Test
922  public void testCreateTableWithEmptyRowInTheSplitKeys() throws IOException{
923    final byte[] tableName = Bytes.toBytes(name.getMethodName());
924    byte[][] splitKeys = new byte[3][];
925    splitKeys[0] = "region1".getBytes();
926    splitKeys[1] = HConstants.EMPTY_BYTE_ARRAY;
927    splitKeys[2] = "region2".getBytes();
928    HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
929    desc.addFamily(new HColumnDescriptor("col"));
930    try {
931      admin.createTable(desc, splitKeys);
932      fail("Test case should fail as empty split key is passed.");
933    } catch (IllegalArgumentException e) {
934      LOG.info("Expected ", e);
935    }
936  }
937
938  @Test
939  public void testTableExist() throws IOException {
940    final TableName table = TableName.valueOf(name.getMethodName());
941    boolean exist;
942    exist = this.admin.tableExists(table);
943    assertEquals(false, exist);
944    TEST_UTIL.createTable(table, HConstants.CATALOG_FAMILY);
945    exist = this.admin.tableExists(table);
946    assertEquals(true, exist);
947  }
948
949  /**
950   * Tests forcing split from client and having scanners successfully ride over split.
951   * @throws Exception
952   * @throws IOException
953   */
954  @Test
955  public void testForceSplit() throws Exception {
956    byte[][] familyNames = new byte[][] { Bytes.toBytes("cf") };
957    int[] rowCounts = new int[] { 6000 };
958    int numVersions = HColumnDescriptor.DEFAULT_VERSIONS;
959    int blockSize = 256;
960    splitTest(null, familyNames, rowCounts, numVersions, blockSize, true);
961
962    byte[] splitKey = Bytes.toBytes(3500);
963    splitTest(splitKey, familyNames, rowCounts, numVersions, blockSize, true);
964    // test regionSplitSync
965    splitTest(splitKey, familyNames, rowCounts, numVersions, blockSize, false);
966  }
967
968  /**
969   * Test retain assignment on enableTable.
970   *
971   * @throws IOException
972   */
973  @Test
974  public void testEnableTableRetainAssignment() throws IOException {
975    final TableName tableName = TableName.valueOf(name.getMethodName());
976    byte[][] splitKeys = { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 },
977        new byte[] { 3, 3, 3 }, new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 },
978        new byte[] { 6, 6, 6 }, new byte[] { 7, 7, 7 }, new byte[] { 8, 8, 8 },
979        new byte[] { 9, 9, 9 } };
980    int expectedRegions = splitKeys.length + 1;
981    HTableDescriptor desc = new HTableDescriptor(tableName);
982    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
983    admin.createTable(desc, splitKeys);
984
985    try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
986      List<HRegionLocation> regions = l.getAllRegionLocations();
987
988      assertEquals(
989          "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
990          expectedRegions, regions.size());
991      // Disable table.
992      admin.disableTable(tableName);
993      // Enable table, use retain assignment to assign regions.
994      admin.enableTable(tableName);
995      List<HRegionLocation> regions2 = l.getAllRegionLocations();
996
997      // Check the assignment.
998      assertEquals(regions.size(), regions2.size());
999      assertTrue(regions2.containsAll(regions));
1000    }
1001  }
1002
1003  /**
1004   * Multi-family scenario. Tests forcing split from client and
1005   * having scanners successfully ride over split.
1006   * @throws Exception
1007   * @throws IOException
1008   */
1009  @Test
1010  public void testForceSplitMultiFamily() throws Exception {
1011    int numVersions = HColumnDescriptor.DEFAULT_VERSIONS;
1012
1013    // use small HFile block size so that we can have lots of blocks in HFile
1014    // Otherwise, if there is only one block,
1015    // HFileBlockIndex.midKey()'s value == startKey
1016    int blockSize = 256;
1017    byte[][] familyNames = new byte[][] { Bytes.toBytes("cf1"),
1018      Bytes.toBytes("cf2") };
1019
1020    // one of the column families isn't splittable
1021    int[] rowCounts = new int[] { 6000, 1 };
1022    splitTest(null, familyNames, rowCounts, numVersions, blockSize, true);
1023
1024    rowCounts = new int[] { 1, 6000 };
1025    splitTest(null, familyNames, rowCounts, numVersions, blockSize, true);
1026
1027    // one column family has much smaller data than the other
1028    // the split key should be based on the largest column family
1029    rowCounts = new int[] { 6000, 300 };
1030    splitTest(null, familyNames, rowCounts, numVersions, blockSize, true);
1031
1032    rowCounts = new int[] { 300, 6000 };
1033    splitTest(null, familyNames, rowCounts, numVersions, blockSize, true);
1034
1035  }
1036
1037  void splitTest(byte[] splitPoint, byte[][] familyNames, int[] rowCounts,
1038    int numVersions, int blockSize, boolean async) throws Exception {
1039    TableName tableName = TableName.valueOf("testForceSplit");
1040    StringBuilder sb = new StringBuilder();
1041    // Add tail to String so can see better in logs where a test is running.
1042    for (int i = 0; i < rowCounts.length; i++) {
1043      sb.append("_").append(Integer.toString(rowCounts[i]));
1044    }
1045    assertFalse(admin.tableExists(tableName));
1046    try (final Table table = TEST_UTIL.createTable(tableName, familyNames,
1047      numVersions, blockSize);
1048      final RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
1049
1050      int rowCount = 0;
1051      byte[] q = new byte[0];
1052
1053      // insert rows into column families. The number of rows that have values
1054      // in a specific column family is decided by rowCounts[familyIndex]
1055      for (int index = 0; index < familyNames.length; index++) {
1056        ArrayList<Put> puts = new ArrayList<>(rowCounts[index]);
1057        for (int i = 0; i < rowCounts[index]; i++) {
1058          byte[] k = Bytes.toBytes(i);
1059          Put put = new Put(k);
1060          put.addColumn(familyNames[index], q, k);
1061          puts.add(put);
1062        }
1063        table.put(puts);
1064
1065        if (rowCount < rowCounts[index]) {
1066          rowCount = rowCounts[index];
1067        }
1068      }
1069
1070      // get the initial layout (should just be one region)
1071      List<HRegionLocation> m = locator.getAllRegionLocations();
1072      LOG.info("Initial regions (" + m.size() + "): " + m);
1073      assertTrue(m.size() == 1);
1074
1075      // Verify row count
1076      Scan scan = new Scan();
1077      ResultScanner scanner = table.getScanner(scan);
1078      int rows = 0;
1079      for (@SuppressWarnings("unused") Result result : scanner) {
1080        rows++;
1081      }
1082      scanner.close();
1083      assertEquals(rowCount, rows);
1084
1085      // Have an outstanding scan going on to make sure we can scan over splits.
1086      scan = new Scan();
1087      scanner = table.getScanner(scan);
1088      // Scan first row so we are into first region before split happens.
1089      scanner.next();
1090
1091      // Split the table
1092      if (async) {
1093        this.admin.split(tableName, splitPoint);
1094        final AtomicInteger count = new AtomicInteger(0);
1095        Thread t = new Thread("CheckForSplit") {
1096          @Override public void run() {
1097            for (int i = 0; i < 45; i++) {
1098              try {
1099                sleep(1000);
1100              } catch (InterruptedException e) {
1101                continue;
1102              }
1103              // check again
1104              List<HRegionLocation> regions = null;
1105              try {
1106                regions = locator.getAllRegionLocations();
1107              } catch (IOException e) {
1108                e.printStackTrace();
1109              }
1110              if (regions == null) continue;
1111              count.set(regions.size());
1112              if (count.get() >= 2) {
1113                LOG.info("Found: " + regions);
1114                break;
1115              }
1116              LOG.debug("Cycle waiting on split");
1117            }
1118            LOG.debug("CheckForSplit thread exited, current region count: " + count.get());
1119          }
1120        };
1121        t.setPriority(Thread.NORM_PRIORITY - 2);
1122        t.start();
1123        t.join();
1124      } else {
1125        // Sync split region, no need to create a thread to check
1126        ((HBaseAdmin)admin).splitRegionSync(m.get(0).getRegionInfo().getRegionName(), splitPoint);
1127      }
1128
1129      // Verify row count
1130      rows = 1; // We counted one row above.
1131      for (@SuppressWarnings("unused") Result result : scanner) {
1132        rows++;
1133        if (rows > rowCount) {
1134          scanner.close();
1135          assertTrue("Scanned more than expected (" + rowCount + ")", false);
1136        }
1137      }
1138      scanner.close();
1139      assertEquals(rowCount, rows);
1140
1141      List<HRegionLocation> regions = null;
1142      try {
1143        regions = locator.getAllRegionLocations();
1144      } catch (IOException e) {
1145        e.printStackTrace();
1146      }
1147      assertEquals(2, regions.size());
1148      if (splitPoint != null) {
1149        // make sure the split point matches our explicit configuration
1150        assertEquals(Bytes.toString(splitPoint),
1151            Bytes.toString(regions.get(0).getRegionInfo().getEndKey()));
1152        assertEquals(Bytes.toString(splitPoint),
1153            Bytes.toString(regions.get(1).getRegionInfo().getStartKey()));
1154        LOG.debug("Properly split on " + Bytes.toString(splitPoint));
1155      } else {
1156        if (familyNames.length > 1) {
1157          int splitKey = Bytes.toInt(regions.get(0).getRegionInfo().getEndKey());
1158          // check if splitKey is based on the largest column family
1159          // in terms of it store size
1160          int deltaForLargestFamily = Math.abs(rowCount / 2 - splitKey);
1161          LOG.debug("SplitKey=" + splitKey + "&deltaForLargestFamily=" + deltaForLargestFamily +
1162              ", r=" + regions.get(0).getRegionInfo());
1163          for (int index = 0; index < familyNames.length; index++) {
1164            int delta = Math.abs(rowCounts[index] / 2 - splitKey);
1165            if (delta < deltaForLargestFamily) {
1166              assertTrue("Delta " + delta + " for family " + index + " should be at least "
1167                      + "deltaForLargestFamily " + deltaForLargestFamily, false);
1168            }
1169          }
1170        }
1171      }
1172      TEST_UTIL.deleteTable(tableName);
1173    }
1174  }
1175
1176  @Test
1177  public void testSplitAndMergeWithReplicaTable() throws Exception {
1178    // The test tries to directly split replica regions and directly merge replica regions. These
1179    // are not allowed. The test validates that. Then the test does a valid split/merge of allowed
1180    // regions.
1181    // Set up a table with 3 regions and replication set to 3
1182    final TableName tableName = TableName.valueOf(name.getMethodName());
1183    HTableDescriptor desc = new HTableDescriptor(tableName);
1184    desc.setRegionReplication(3);
1185    byte[] cf = "f".getBytes();
1186    HColumnDescriptor hcd = new HColumnDescriptor(cf);
1187    desc.addFamily(hcd);
1188    byte[][] splitRows = new byte[2][];
1189    splitRows[0] = new byte[]{(byte)'4'};
1190    splitRows[1] = new byte[]{(byte)'7'};
1191    TEST_UTIL.getAdmin().createTable(desc, splitRows);
1192    List<HRegion> oldRegions;
1193    do {
1194      oldRegions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
1195      Thread.sleep(10);
1196    } while (oldRegions.size() != 9); //3 regions * 3 replicas
1197    // write some data to the table
1198    Table ht = TEST_UTIL.getConnection().getTable(tableName);
1199    List<Put> puts = new ArrayList<>();
1200    byte[] qualifier = "c".getBytes();
1201    Put put = new Put(new byte[]{(byte)'1'});
1202    put.addColumn(cf, qualifier, "100".getBytes());
1203    puts.add(put);
1204    put = new Put(new byte[]{(byte)'6'});
1205    put.addColumn(cf, qualifier, "100".getBytes());
1206    puts.add(put);
1207    put = new Put(new byte[]{(byte)'8'});
1208    put.addColumn(cf, qualifier, "100".getBytes());
1209    puts.add(put);
1210    ht.put(puts);
1211    ht.close();
1212    List<Pair<RegionInfo, ServerName>> regions =
1213        MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getConnection(), tableName);
1214    boolean gotException = false;
1215    // the element at index 1 would be a replica (since the metareader gives us ordered
1216    // regions). Try splitting that region via the split API . Should fail
1217    try {
1218      TEST_UTIL.getAdmin().splitRegionAsync(regions.get(1).getFirst().getRegionName()).get();
1219    } catch (IllegalArgumentException ex) {
1220      gotException = true;
1221    }
1222    assertTrue(gotException);
1223    gotException = false;
1224    // the element at index 1 would be a replica (since the metareader gives us ordered
1225    // regions). Try splitting that region via a different split API (the difference is
1226    // this API goes direct to the regionserver skipping any checks in the admin). Should fail
1227    try {
1228      TEST_UTIL.getHBaseAdmin().splitRegionAsync(regions.get(1).getFirst(),
1229          new byte[]{(byte)'1'});
1230    } catch (IOException ex) {
1231      gotException = true;
1232    }
1233    assertTrue(gotException);
1234
1235    gotException = false;
1236    //testing Sync split operation
1237    try {
1238      TEST_UTIL.getHBaseAdmin().splitRegionSync(regions.get(1).getFirst().getRegionName(),
1239          new byte[]{(byte)'1'});
1240    } catch (IllegalArgumentException ex) {
1241      gotException = true;
1242    }
1243    assertTrue(gotException);
1244
1245    gotException = false;
1246    // Try merging a replica with another. Should fail.
1247    try {
1248      TEST_UTIL.getHBaseAdmin().mergeRegionsSync(
1249        regions.get(1).getFirst().getEncodedNameAsBytes(),
1250        regions.get(2).getFirst().getEncodedNameAsBytes(),
1251        true);
1252    } catch (IllegalArgumentException m) {
1253      gotException = true;
1254    }
1255    assertTrue(gotException);
1256    // Try going to the master directly (that will skip the check in admin)
1257    try {
1258      byte[][] nameofRegionsToMerge = new byte[2][];
1259      nameofRegionsToMerge[0] =  regions.get(1).getFirst().getEncodedNameAsBytes();
1260      nameofRegionsToMerge[1] = regions.get(2).getFirst().getEncodedNameAsBytes();
1261      MergeTableRegionsRequest request = RequestConverter
1262          .buildMergeTableRegionsRequest(
1263            nameofRegionsToMerge,
1264            true,
1265            HConstants.NO_NONCE,
1266            HConstants.NO_NONCE);
1267      ((ClusterConnection) TEST_UTIL.getAdmin().getConnection()).getMaster()
1268        .mergeTableRegions(null, request);
1269    } catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException m) {
1270      Throwable t = m.getCause();
1271      do {
1272        if (t instanceof MergeRegionException) {
1273          gotException = true;
1274          break;
1275        }
1276        t = t.getCause();
1277      } while (t != null);
1278    }
1279    assertTrue(gotException);
1280  }
1281
1282  @Test (expected=IllegalArgumentException.class)
1283  public void testInvalidHColumnDescriptor() throws IOException {
1284     new HColumnDescriptor("/cfamily/name");
1285  }
1286
1287  @Test
1288  public void testEnableDisableAddColumnDeleteColumn() throws Exception {
1289    final TableName tableName = TableName.valueOf(name.getMethodName());
1290    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
1291    while (!this.admin.isTableEnabled(TableName.valueOf(name.getMethodName()))) {
1292      Thread.sleep(10);
1293    }
1294    this.admin.disableTable(tableName);
1295    try {
1296      TEST_UTIL.getConnection().getTable(tableName);
1297    } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
1298      //expected
1299    }
1300
1301    this.admin.addColumnFamily(tableName, new HColumnDescriptor("col2"));
1302    this.admin.enableTable(tableName);
1303    try {
1304      this.admin.deleteColumnFamily(tableName, Bytes.toBytes("col2"));
1305    } catch (TableNotDisabledException e) {
1306      LOG.info(e.toString(), e);
1307    }
1308    this.admin.disableTable(tableName);
1309    this.admin.deleteTable(tableName);
1310  }
1311
1312  @Test
1313  public void testDeleteLastColumnFamily() throws Exception {
1314    final TableName tableName = TableName.valueOf(name.getMethodName());
1315    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
1316    while (!this.admin.isTableEnabled(TableName.valueOf(name.getMethodName()))) {
1317      Thread.sleep(10);
1318    }
1319
1320    // test for enabled table
1321    try {
1322      this.admin.deleteColumnFamily(tableName, HConstants.CATALOG_FAMILY);
1323      fail("Should have failed to delete the only column family of a table");
1324    } catch (InvalidFamilyOperationException ex) {
1325      // expected
1326    }
1327
1328    // test for disabled table
1329    this.admin.disableTable(tableName);
1330
1331    try {
1332      this.admin.deleteColumnFamily(tableName, HConstants.CATALOG_FAMILY);
1333      fail("Should have failed to delete the only column family of a table");
1334    } catch (InvalidFamilyOperationException ex) {
1335      // expected
1336    }
1337
1338    this.admin.deleteTable(tableName);
1339  }
1340
1341  /*
1342   * Test DFS replication for column families, where one CF has default replication(3) and the other
1343   * is set to 1.
1344   */
1345  @Test
1346  public void testHFileReplication() throws Exception {
1347    final TableName tableName = TableName.valueOf(this.name.getMethodName());
1348    String fn1 = "rep1";
1349    HColumnDescriptor hcd1 = new HColumnDescriptor(fn1);
1350    hcd1.setDFSReplication((short) 1);
1351    String fn = "defaultRep";
1352    HColumnDescriptor hcd = new HColumnDescriptor(fn);
1353    HTableDescriptor htd = new HTableDescriptor(tableName);
1354    htd.addFamily(hcd);
1355    htd.addFamily(hcd1);
1356    Table table = TEST_UTIL.createTable(htd, null);
1357    TEST_UTIL.waitTableAvailable(tableName);
1358    Put p = new Put(Bytes.toBytes("defaultRep_rk"));
1359    byte[] q1 = Bytes.toBytes("q1");
1360    byte[] v1 = Bytes.toBytes("v1");
1361    p.addColumn(Bytes.toBytes(fn), q1, v1);
1362    List<Put> puts = new ArrayList<>(2);
1363    puts.add(p);
1364    p = new Put(Bytes.toBytes("rep1_rk"));
1365    p.addColumn(Bytes.toBytes(fn1), q1, v1);
1366    puts.add(p);
1367    try {
1368      table.put(puts);
1369      admin.flush(tableName);
1370
1371      List<HRegion> regions = TEST_UTIL.getMiniHBaseCluster().getRegions(tableName);
1372      for (HRegion r : regions) {
1373        HStore store = r.getStore(Bytes.toBytes(fn));
1374        for (HStoreFile sf : store.getStorefiles()) {
1375          assertTrue(sf.toString().contains(fn));
1376          assertTrue("Column family " + fn + " should have 3 copies",
1377            FSUtils.getDefaultReplication(TEST_UTIL.getTestFileSystem(), sf.getPath()) == (sf
1378                .getFileInfo().getFileStatus().getReplication()));
1379        }
1380
1381        store = r.getStore(Bytes.toBytes(fn1));
1382        for (HStoreFile sf : store.getStorefiles()) {
1383          assertTrue(sf.toString().contains(fn1));
1384          assertTrue("Column family " + fn1 + " should have only 1 copy", 1 == sf.getFileInfo()
1385              .getFileStatus().getReplication());
1386        }
1387      }
1388    } finally {
1389      if (admin.isTableEnabled(tableName)) {
1390        this.admin.disableTable(tableName);
1391        this.admin.deleteTable(tableName);
1392      }
1393    }
1394  }
1395
1396  @Test
1397  public void testMergeRegions() throws Exception {
1398    final TableName tableName = TableName.valueOf(name.getMethodName());
1399    HColumnDescriptor cd = new HColumnDescriptor("d");
1400    HTableDescriptor td = new HTableDescriptor(tableName);
1401    td.addFamily(cd);
1402    byte[][] splitRows = new byte[2][];
1403    splitRows[0] = new byte[]{(byte)'3'};
1404    splitRows[1] = new byte[]{(byte)'6'};
1405    try {
1406      TEST_UTIL.createTable(td, splitRows);
1407      TEST_UTIL.waitTableAvailable(tableName);
1408
1409      List<RegionInfo> tableRegions;
1410      RegionInfo regionA;
1411      RegionInfo regionB;
1412
1413      // merge with full name
1414      tableRegions = admin.getRegions(tableName);
1415      assertEquals(3, admin.getTableRegions(tableName).size());
1416      regionA = tableRegions.get(0);
1417      regionB = tableRegions.get(1);
1418      // TODO convert this to version that is synchronous (See HBASE-16668)
1419      admin.mergeRegionsAsync(regionA.getRegionName(), regionB.getRegionName(), false)
1420          .get(60, TimeUnit.SECONDS);
1421
1422      assertEquals(2, admin.getTableRegions(tableName).size());
1423
1424      // merge with encoded name
1425      tableRegions = admin.getRegions(tableName);
1426      regionA = tableRegions.get(0);
1427      regionB = tableRegions.get(1);
1428      // TODO convert this to version that is synchronous (See HBASE-16668)
1429      admin.mergeRegionsAsync(
1430        regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false)
1431          .get(60, TimeUnit.SECONDS);
1432
1433      assertEquals(1, admin.getTableRegions(tableName).size());
1434    } finally {
1435      this.admin.disableTable(tableName);
1436      this.admin.deleteTable(tableName);
1437    }
1438  }
1439
1440  @Test
1441  public void testMergeRegionsInvalidRegionCount()
1442      throws IOException, InterruptedException, ExecutionException {
1443    TableName tableName = TableName.valueOf(name.getMethodName());
1444    TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName)
1445      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("d")).build();
1446    byte[][] splitRows = new byte[2][];
1447    splitRows[0] = new byte[] { (byte) '3' };
1448    splitRows[1] = new byte[] { (byte) '6' };
1449    try {
1450      TEST_UTIL.createTable(td, splitRows);
1451      TEST_UTIL.waitTableAvailable(tableName);
1452
1453      List<RegionInfo> tableRegions = admin.getRegions(tableName);
1454      // 0
1455      try {
1456        admin.mergeRegionsAsync(new byte[0][0], false).get();
1457        fail();
1458      } catch (IllegalArgumentException e) {
1459        // expected
1460      }
1461      // 1
1462      try {
1463        admin.mergeRegionsAsync(new byte[][] { tableRegions.get(0).getEncodedNameAsBytes() }, false)
1464          .get();
1465        fail();
1466      } catch (IllegalArgumentException e) {
1467        // expected
1468      }
1469    } finally {
1470      admin.disableTable(tableName);
1471      admin.deleteTable(tableName);
1472    }
1473  }
1474
1475  @Test
1476  public void testSplitShouldNotHappenIfSplitIsDisabledForTable()
1477      throws Exception {
1478    final TableName tableName = TableName.valueOf(name.getMethodName());
1479    HTableDescriptor htd = new HTableDescriptor(tableName);
1480    htd.addFamily(new HColumnDescriptor("f"));
1481    htd.setRegionSplitPolicyClassName(DisabledRegionSplitPolicy.class.getName());
1482    Table table = TEST_UTIL.createTable(htd, null);
1483    for(int i = 0; i < 10; i++) {
1484      Put p = new Put(Bytes.toBytes("row"+i));
1485      byte[] q1 = Bytes.toBytes("q1");
1486      byte[] v1 = Bytes.toBytes("v1");
1487      p.addColumn(Bytes.toBytes("f"), q1, v1);
1488      table.put(p);
1489    }
1490    this.admin.flush(tableName);
1491    try {
1492      this.admin.split(tableName, Bytes.toBytes("row5"));
1493      Threads.sleep(10000);
1494    } catch (Exception e) {
1495      // Nothing to do.
1496    }
1497    // Split should not happen.
1498    List<RegionInfo> allRegions = MetaTableAccessor.getTableRegions(
1499        this.admin.getConnection(), tableName, true);
1500    assertEquals(1, allRegions.size());
1501  }
1502
1503  @Test
1504  public void testCloneTableSchema() throws Exception {
1505    final TableName tableName = TableName.valueOf(name.getMethodName());
1506    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
1507    testCloneTableSchema(tableName, newTableName, false);
1508  }
1509
1510  @Test
1511  public void testCloneTableSchemaPreservingSplits() throws Exception {
1512    final TableName tableName = TableName.valueOf(name.getMethodName());
1513    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
1514    testCloneTableSchema(tableName, newTableName, true);
1515  }
1516
1517  private void testCloneTableSchema(final TableName tableName,
1518      final TableName newTableName, boolean preserveSplits) throws Exception {
1519    byte[] FAMILY_0 = Bytes.toBytes("cf0");
1520    byte[] FAMILY_1 = Bytes.toBytes("cf1");
1521    byte[][] splitKeys = new byte[2][];
1522    splitKeys[0] = Bytes.toBytes(4);
1523    splitKeys[1] = Bytes.toBytes(8);
1524    int NUM_FAMILYS = 2;
1525    int NUM_REGIONS = 3;
1526    int BLOCK_SIZE = 1024;
1527    int TTL = 86400;
1528    boolean BLOCK_CACHE = false;
1529
1530    // Create the table
1531    TableDescriptor tableDesc = TableDescriptorBuilder
1532        .newBuilder(tableName)
1533        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0))
1534        .setColumnFamily(ColumnFamilyDescriptorBuilder
1535            .newBuilder(FAMILY_1)
1536            .setBlocksize(BLOCK_SIZE)
1537            .setBlockCacheEnabled(BLOCK_CACHE)
1538            .setTimeToLive(TTL)
1539            .build()
1540        ).build();
1541    admin.createTable(tableDesc, splitKeys);
1542
1543    assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
1544    assertTrue("Table should be created with splitKyes + 1 rows in META",
1545        admin.isTableAvailable(tableName, splitKeys));
1546
1547    // clone & Verify
1548    admin.cloneTableSchema(tableName, newTableName, preserveSplits);
1549    TableDescriptor newTableDesc = admin.getDescriptor(newTableName);
1550
1551    assertEquals(NUM_FAMILYS, newTableDesc.getColumnFamilyCount());
1552    assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize());
1553    assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled());
1554    assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive());
1555    TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc);
1556
1557    if (preserveSplits) {
1558      assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size());
1559      assertTrue("New table should be created with splitKyes + 1 rows in META",
1560          admin.isTableAvailable(newTableName, splitKeys));
1561    } else {
1562      assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size());
1563    }
1564  }
1565
1566  @Test
1567  public void testCloneTableSchemaWithNonExistentSourceTable() throws Exception {
1568    final TableName tableName = TableName.valueOf(name.getMethodName());
1569    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
1570    // test for non-existent source table
1571    try {
1572      admin.cloneTableSchema(tableName, newTableName, false);
1573      fail("Should have failed to create a new table by cloning non-existent source table.");
1574    } catch (TableNotFoundException ex) {
1575      // expected
1576    }
1577  }
1578
1579  @Test
1580  public void testCloneTableSchemaWithExistentDestinationTable() throws Exception {
1581    final TableName tableName = TableName.valueOf(name.getMethodName());
1582    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
1583    byte[] FAMILY_0 = Bytes.toBytes("cf0");
1584    TEST_UTIL.createTable(tableName, FAMILY_0);
1585    TEST_UTIL.createTable(newTableName, FAMILY_0);
1586    // test for existent destination table
1587    try {
1588      admin.cloneTableSchema(tableName, newTableName, false);
1589      fail("Should have failed to create a existent table.");
1590    } catch (TableExistsException ex) {
1591      // expected
1592    }
1593  }
1594
1595  @Test
1596  public void testModifyTableOnTableWithRegionReplicas() throws Exception {
1597    TableName tableName = TableName.valueOf(name.getMethodName());
1598    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
1599        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf")))
1600        .setRegionReplication(5)
1601        .build();
1602
1603    admin.createTable(desc);
1604
1605    int maxFileSize = 10000000;
1606    TableDescriptor newDesc = TableDescriptorBuilder.newBuilder(desc)
1607        .setMaxFileSize(maxFileSize)
1608        .build();
1609
1610    admin.modifyTable(newDesc);
1611    TableDescriptor newTableDesc = admin.getDescriptor(tableName);
1612    assertEquals(maxFileSize, newTableDesc.getMaxFileSize());
1613  }
1614}