001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.client; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertNotNull; 023import static org.junit.Assert.assertTrue; 024import static org.junit.Assert.fail; 025 026import java.io.IOException; 027import java.util.ArrayList; 028import java.util.HashMap; 029import java.util.Iterator; 030import java.util.List; 031import java.util.Map; 032import java.util.concurrent.ExecutionException; 033import java.util.concurrent.TimeUnit; 034import java.util.concurrent.atomic.AtomicInteger; 035import org.apache.hadoop.hbase.HBaseClassTestRule; 036import org.apache.hadoop.hbase.HBaseTestingUtility; 037import org.apache.hadoop.hbase.HColumnDescriptor; 038import org.apache.hadoop.hbase.HConstants; 039import org.apache.hadoop.hbase.HRegionLocation; 040import org.apache.hadoop.hbase.HTableDescriptor; 041import org.apache.hadoop.hbase.InvalidFamilyOperationException; 042import org.apache.hadoop.hbase.MetaTableAccessor; 043import org.apache.hadoop.hbase.ServerName; 044import org.apache.hadoop.hbase.TableExistsException; 045import org.apache.hadoop.hbase.TableName; 046import org.apache.hadoop.hbase.TableNotDisabledException; 047import org.apache.hadoop.hbase.TableNotEnabledException; 048import org.apache.hadoop.hbase.TableNotFoundException; 049import org.apache.hadoop.hbase.exceptions.MergeRegionException; 050import org.apache.hadoop.hbase.master.LoadBalancer; 051import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy; 052import org.apache.hadoop.hbase.regionserver.HRegion; 053import org.apache.hadoop.hbase.regionserver.HStore; 054import org.apache.hadoop.hbase.regionserver.HStoreFile; 055import org.apache.hadoop.hbase.testclassification.ClientTests; 056import org.apache.hadoop.hbase.testclassification.LargeTests; 057import org.apache.hadoop.hbase.util.Bytes; 058import org.apache.hadoop.hbase.util.FSUtils; 059import org.apache.hadoop.hbase.util.Pair; 060import org.apache.hadoop.hbase.util.Threads; 061import org.junit.After; 062import org.junit.AfterClass; 063import org.junit.Before; 064import org.junit.BeforeClass; 065import org.junit.ClassRule; 066import org.junit.Rule; 067import org.junit.Test; 068import org.junit.experimental.categories.Category; 069import org.junit.rules.TestName; 070import org.slf4j.Logger; 071import org.slf4j.LoggerFactory; 072 073import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; 074import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsRequest; 075 076/** 077 * Class to test HBaseAdmin. 078 * Spins up the minicluster once at test start and then takes it down afterward. 079 * Add any testing of HBaseAdmin functionality here. 080 */ 081@Category({LargeTests.class, ClientTests.class}) 082public class TestAdmin1 { 083 084 @ClassRule 085 public static final HBaseClassTestRule CLASS_RULE = 086 HBaseClassTestRule.forClass(TestAdmin1.class); 087 088 private static final Logger LOG = LoggerFactory.getLogger(TestAdmin1.class); 089 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 090 private Admin admin; 091 092 @Rule 093 public TestName name = new TestName(); 094 095 @BeforeClass 096 public static void setUpBeforeClass() throws Exception { 097 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); 098 TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); 099 TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6); 100 TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); 101 TEST_UTIL.startMiniCluster(3); 102 } 103 104 @AfterClass 105 public static void tearDownAfterClass() throws Exception { 106 TEST_UTIL.shutdownMiniCluster(); 107 } 108 109 @Before 110 public void setUp() throws Exception { 111 this.admin = TEST_UTIL.getAdmin(); 112 } 113 114 @After 115 public void tearDown() throws Exception { 116 for (HTableDescriptor htd : this.admin.listTables()) { 117 TEST_UTIL.deleteTable(htd.getTableName()); 118 } 119 } 120 121 @Test 122 public void testSplitFlushCompactUnknownTable() throws InterruptedException { 123 final TableName unknowntable = TableName.valueOf(name.getMethodName()); 124 Exception exception = null; 125 try { 126 this.admin.compact(unknowntable); 127 } catch (IOException e) { 128 exception = e; 129 } 130 assertTrue(exception instanceof TableNotFoundException); 131 132 exception = null; 133 try { 134 this.admin.flush(unknowntable); 135 } catch (IOException e) { 136 exception = e; 137 } 138 assertTrue(exception instanceof TableNotFoundException); 139 140 exception = null; 141 try { 142 this.admin.split(unknowntable); 143 } catch (IOException e) { 144 exception = e; 145 } 146 assertTrue(exception instanceof TableNotFoundException); 147 } 148 149 @Test 150 public void testDeleteEditUnknownColumnFamilyAndOrTable() throws IOException { 151 // Test we get exception if we try to 152 final TableName nonexistentTable = TableName.valueOf("nonexistent"); 153 final byte[] nonexistentColumn = Bytes.toBytes("nonexistent"); 154 HColumnDescriptor nonexistentHcd = new HColumnDescriptor(nonexistentColumn); 155 Exception exception = null; 156 try { 157 this.admin.addColumnFamily(nonexistentTable, nonexistentHcd); 158 } catch (IOException e) { 159 exception = e; 160 } 161 assertTrue(exception instanceof TableNotFoundException); 162 163 exception = null; 164 try { 165 this.admin.deleteTable(nonexistentTable); 166 } catch (IOException e) { 167 exception = e; 168 } 169 assertTrue(exception instanceof TableNotFoundException); 170 171 exception = null; 172 try { 173 this.admin.deleteColumnFamily(nonexistentTable, nonexistentColumn); 174 } catch (IOException e) { 175 exception = e; 176 } 177 assertTrue(exception instanceof TableNotFoundException); 178 179 exception = null; 180 try { 181 this.admin.disableTable(nonexistentTable); 182 } catch (IOException e) { 183 exception = e; 184 } 185 assertTrue(exception instanceof TableNotFoundException); 186 187 exception = null; 188 try { 189 this.admin.enableTable(nonexistentTable); 190 } catch (IOException e) { 191 exception = e; 192 } 193 assertTrue(exception instanceof TableNotFoundException); 194 195 exception = null; 196 try { 197 this.admin.modifyColumnFamily(nonexistentTable, nonexistentHcd); 198 } catch (IOException e) { 199 exception = e; 200 } 201 assertTrue(exception instanceof TableNotFoundException); 202 203 exception = null; 204 try { 205 HTableDescriptor htd = new HTableDescriptor(nonexistentTable); 206 htd.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); 207 this.admin.modifyTable(htd.getTableName(), htd); 208 } catch (IOException e) { 209 exception = e; 210 } 211 assertTrue(exception instanceof TableNotFoundException); 212 213 // Now make it so at least the table exists and then do tests against a 214 // nonexistent column family -- see if we get right exceptions. 215 final TableName tableName = TableName.valueOf(name.getMethodName() + System.currentTimeMillis()); 216 HTableDescriptor htd = new HTableDescriptor(tableName); 217 htd.addFamily(new HColumnDescriptor("cf")); 218 this.admin.createTable(htd); 219 try { 220 exception = null; 221 try { 222 this.admin.deleteColumnFamily(htd.getTableName(), nonexistentHcd.getName()); 223 } catch (IOException e) { 224 exception = e; 225 } 226 assertTrue("found=" + exception.getClass().getName(), 227 exception instanceof InvalidFamilyOperationException); 228 229 exception = null; 230 try { 231 this.admin.modifyColumnFamily(htd.getTableName(), nonexistentHcd); 232 } catch (IOException e) { 233 exception = e; 234 } 235 assertTrue("found=" + exception.getClass().getName(), 236 exception instanceof InvalidFamilyOperationException); 237 } finally { 238 this.admin.disableTable(tableName); 239 this.admin.deleteTable(tableName); 240 } 241 } 242 243 @Test 244 public void testDisableAndEnableTable() throws IOException { 245 final byte [] row = Bytes.toBytes("row"); 246 final byte [] qualifier = Bytes.toBytes("qualifier"); 247 final byte [] value = Bytes.toBytes("value"); 248 final TableName table = TableName.valueOf(name.getMethodName()); 249 Table ht = TEST_UTIL.createTable(table, HConstants.CATALOG_FAMILY); 250 Put put = new Put(row); 251 put.addColumn(HConstants.CATALOG_FAMILY, qualifier, value); 252 ht.put(put); 253 Get get = new Get(row); 254 get.addColumn(HConstants.CATALOG_FAMILY, qualifier); 255 ht.get(get); 256 257 this.admin.disableTable(ht.getName()); 258 assertTrue("Table must be disabled.", TEST_UTIL.getHBaseCluster() 259 .getMaster().getTableStateManager().isTableState( 260 ht.getName(), TableState.State.DISABLED)); 261 assertEquals(TableState.State.DISABLED, getStateFromMeta(table)); 262 263 // Test that table is disabled 264 get = new Get(row); 265 get.addColumn(HConstants.CATALOG_FAMILY, qualifier); 266 boolean ok = false; 267 try { 268 ht.get(get); 269 } catch (TableNotEnabledException e) { 270 ok = true; 271 } 272 ok = false; 273 // verify that scan encounters correct exception 274 Scan scan = new Scan(); 275 try { 276 ResultScanner scanner = ht.getScanner(scan); 277 Result res = null; 278 do { 279 res = scanner.next(); 280 } while (res != null); 281 } catch (TableNotEnabledException e) { 282 ok = true; 283 } 284 assertTrue(ok); 285 this.admin.enableTable(table); 286 assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster() 287 .getMaster().getTableStateManager().isTableState( 288 ht.getName(), TableState.State.ENABLED)); 289 assertEquals(TableState.State.ENABLED, getStateFromMeta(table)); 290 291 // Test that table is enabled 292 try { 293 ht.get(get); 294 } catch (RetriesExhaustedException e) { 295 ok = false; 296 } 297 assertTrue(ok); 298 ht.close(); 299 } 300 301 private TableState.State getStateFromMeta(TableName table) throws IOException { 302 TableState state = 303 MetaTableAccessor.getTableState(TEST_UTIL.getConnection(), table); 304 assertNotNull(state); 305 return state.getState(); 306 } 307 308 @Test 309 public void testDisableAndEnableTables() throws IOException { 310 final byte [] row = Bytes.toBytes("row"); 311 final byte [] qualifier = Bytes.toBytes("qualifier"); 312 final byte [] value = Bytes.toBytes("value"); 313 final TableName table1 = TableName.valueOf(name.getMethodName() + "1"); 314 final TableName table2 = TableName.valueOf(name.getMethodName() + "2"); 315 Table ht1 = TEST_UTIL.createTable(table1, HConstants.CATALOG_FAMILY); 316 Table ht2 = TEST_UTIL.createTable(table2, HConstants.CATALOG_FAMILY); 317 Put put = new Put(row); 318 put.addColumn(HConstants.CATALOG_FAMILY, qualifier, value); 319 ht1.put(put); 320 ht2.put(put); 321 Get get = new Get(row); 322 get.addColumn(HConstants.CATALOG_FAMILY, qualifier); 323 ht1.get(get); 324 ht2.get(get); 325 326 this.admin.disableTables("testDisableAndEnableTable.*"); 327 328 // Test that tables are disabled 329 get = new Get(row); 330 get.addColumn(HConstants.CATALOG_FAMILY, qualifier); 331 boolean ok = false; 332 try { 333 ht1.get(get); 334 ht2.get(get); 335 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) { 336 ok = true; 337 } 338 339 assertEquals(TableState.State.DISABLED, getStateFromMeta(table1)); 340 assertEquals(TableState.State.DISABLED, getStateFromMeta(table2)); 341 342 343 assertTrue(ok); 344 this.admin.enableTables("testDisableAndEnableTable.*"); 345 346 // Test that tables are enabled 347 try { 348 ht1.get(get); 349 } catch (IOException e) { 350 ok = false; 351 } 352 try { 353 ht2.get(get); 354 } catch (IOException e) { 355 ok = false; 356 } 357 assertTrue(ok); 358 359 ht1.close(); 360 ht2.close(); 361 362 assertEquals(TableState.State.ENABLED, getStateFromMeta(table1)); 363 assertEquals(TableState.State.ENABLED, getStateFromMeta(table2)); 364 } 365 366 @Test 367 public void testCreateTable() throws IOException { 368 HTableDescriptor [] tables = admin.listTables(); 369 int numTables = tables.length; 370 final TableName tableName = TableName.valueOf(name.getMethodName()); 371 TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); 372 tables = this.admin.listTables(); 373 assertEquals(numTables + 1, tables.length); 374 assertTrue("Table must be enabled.", 375 TEST_UTIL.getHBaseCluster().getMaster().getTableStateManager() 376 .isTableState(tableName, TableState.State.ENABLED)); 377 assertEquals(TableState.State.ENABLED, getStateFromMeta(tableName)); 378 } 379 380 @Test 381 public void testTruncateTable() throws IOException { 382 testTruncateTable(TableName.valueOf(name.getMethodName()), false); 383 } 384 385 @Test 386 public void testTruncateTablePreservingSplits() throws IOException { 387 testTruncateTable(TableName.valueOf(name.getMethodName()), true); 388 } 389 390 private void testTruncateTable(final TableName tableName, boolean preserveSplits) 391 throws IOException { 392 byte[][] splitKeys = new byte[2][]; 393 splitKeys[0] = Bytes.toBytes(4); 394 splitKeys[1] = Bytes.toBytes(8); 395 396 // Create & Fill the table 397 Table table = TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY, splitKeys); 398 try { 399 TEST_UTIL.loadNumericRows(table, HConstants.CATALOG_FAMILY, 0, 10); 400 assertEquals(10, TEST_UTIL.countRows(table)); 401 } finally { 402 table.close(); 403 } 404 assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size()); 405 406 // Truncate & Verify 407 this.admin.disableTable(tableName); 408 this.admin.truncateTable(tableName, preserveSplits); 409 table = TEST_UTIL.getConnection().getTable(tableName); 410 try { 411 assertEquals(0, TEST_UTIL.countRows(table)); 412 } finally { 413 table.close(); 414 } 415 if (preserveSplits) { 416 assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size()); 417 } else { 418 assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(tableName).size()); 419 } 420 } 421 422 @Test 423 public void testGetTableDescriptor() throws IOException { 424 HColumnDescriptor fam1 = new HColumnDescriptor("fam1"); 425 HColumnDescriptor fam2 = new HColumnDescriptor("fam2"); 426 HColumnDescriptor fam3 = new HColumnDescriptor("fam3"); 427 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); 428 htd.addFamily(fam1); 429 htd.addFamily(fam2); 430 htd.addFamily(fam3); 431 this.admin.createTable(htd); 432 Table table = TEST_UTIL.getConnection().getTable(htd.getTableName()); 433 TableDescriptor confirmedHtd = table.getDescriptor(); 434 assertEquals(0, TableDescriptor.COMPARATOR.compare(htd, confirmedHtd)); 435 MetaTableAccessor.fullScanMetaAndPrint(TEST_UTIL.getConnection()); 436 table.close(); 437 } 438 439 @Test 440 public void testCompactionTimestamps() throws Exception { 441 HColumnDescriptor fam1 = new HColumnDescriptor("fam1"); 442 final TableName tableName = TableName.valueOf(name.getMethodName()); 443 HTableDescriptor htd = new HTableDescriptor(tableName); 444 htd.addFamily(fam1); 445 this.admin.createTable(htd); 446 Table table = TEST_UTIL.getConnection().getTable(htd.getTableName()); 447 long ts = this.admin.getLastMajorCompactionTimestamp(tableName); 448 assertEquals(0, ts); 449 Put p = new Put(Bytes.toBytes("row1")); 450 p.addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("fam1"), Bytes.toBytes("fam1")); 451 table.put(p); 452 ts = this.admin.getLastMajorCompactionTimestamp(tableName); 453 // no files written -> no data 454 assertEquals(0, ts); 455 456 this.admin.flush(tableName); 457 ts = this.admin.getLastMajorCompactionTimestamp(tableName); 458 // still 0, we flushed a file, but no major compaction happened 459 assertEquals(0, ts); 460 461 byte[] regionName; 462 try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) { 463 regionName = l.getAllRegionLocations().get(0).getRegionInfo().getRegionName(); 464 } 465 long ts1 = this.admin.getLastMajorCompactionTimestampForRegion(regionName); 466 assertEquals(ts, ts1); 467 p = new Put(Bytes.toBytes("row2")); 468 p.addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("fam1"), Bytes.toBytes("fam1")); 469 table.put(p); 470 this.admin.flush(tableName); 471 ts = this.admin.getLastMajorCompactionTimestamp(tableName); 472 // make sure the region API returns the same value, as the old file is still around 473 assertEquals(ts1, ts); 474 475 TEST_UTIL.compact(tableName, true); 476 table.put(p); 477 // forces a wait for the compaction 478 this.admin.flush(tableName); 479 ts = this.admin.getLastMajorCompactionTimestamp(tableName); 480 // after a compaction our earliest timestamp will have progressed forward 481 assertTrue(ts > ts1); 482 483 // region api still the same 484 ts1 = this.admin.getLastMajorCompactionTimestampForRegion(regionName); 485 assertEquals(ts, ts1); 486 table.put(p); 487 this.admin.flush(tableName); 488 ts = this.admin.getLastMajorCompactionTimestamp(tableName); 489 assertEquals(ts, ts1); 490 table.close(); 491 } 492 493 @Test 494 public void testHColumnValidName() { 495 boolean exceptionThrown; 496 try { 497 new HColumnDescriptor("\\test\\abc"); 498 } catch(IllegalArgumentException iae) { 499 exceptionThrown = true; 500 assertTrue(exceptionThrown); 501 } 502 } 503 504 /** 505 * Verify schema change for read only table 506 */ 507 @Test 508 public void testReadOnlyTableModify() throws IOException, InterruptedException { 509 final TableName tableName = TableName.valueOf(name.getMethodName()); 510 TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); 511 512 // Make table read only 513 TableDescriptor htd = TableDescriptorBuilder.newBuilder(this.admin.getDescriptor(tableName)) 514 .setReadOnly(true).build(); 515 admin.modifyTable(htd); 516 517 // try to modify the read only table now 518 htd = TableDescriptorBuilder.newBuilder(this.admin.getDescriptor(tableName)) 519 .setCompactionEnabled(false).build(); 520 admin.modifyTable(htd); 521 // Delete the table 522 this.admin.disableTable(tableName); 523 this.admin.deleteTable(tableName); 524 assertFalse(this.admin.tableExists(tableName)); 525 } 526 527 @Test(expected = TableNotDisabledException.class) 528 public void testModifyRegionReplicasEnabledTable() throws Exception { 529 final TableName tableName = TableName.valueOf(name.getMethodName()); 530 TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); 531 532 // Modify region replication count 533 TableDescriptor htd = TableDescriptorBuilder.newBuilder(admin.getDescriptor(tableName)) 534 .setRegionReplication(3).build(); 535 try { 536 // try to modify the region replication count without disabling the table 537 admin.modifyTable(htd); 538 fail("Expected an exception"); 539 } finally { 540 // Delete the table 541 admin.disableTable(tableName); 542 admin.deleteTable(tableName); 543 assertFalse(admin.tableExists(tableName)); 544 } 545 } 546 547 /** 548 * Verify schema modification takes. 549 */ 550 @Test 551 public void testOnlineChangeTableSchema() throws IOException, InterruptedException { 552 final TableName tableName = TableName.valueOf(name.getMethodName()); 553 HTableDescriptor [] tables = admin.listTables(); 554 int numTables = tables.length; 555 TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); 556 tables = this.admin.listTables(); 557 assertEquals(numTables + 1, tables.length); 558 559 // FIRST, do htabledescriptor changes. 560 HTableDescriptor htd = this.admin.getTableDescriptor(tableName); 561 // Make a copy and assert copy is good. 562 HTableDescriptor copy = new HTableDescriptor(htd); 563 assertTrue(htd.equals(copy)); 564 // Now amend the copy. Introduce differences. 565 long newFlushSize = htd.getMemStoreFlushSize() / 2; 566 if (newFlushSize <=0) { 567 newFlushSize = HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE / 2; 568 } 569 copy.setMemStoreFlushSize(newFlushSize); 570 final String key = "anyoldkey"; 571 assertTrue(htd.getValue(key) == null); 572 copy.setValue(key, key); 573 boolean expectedException = false; 574 try { 575 admin.modifyTable(tableName, copy); 576 } catch (TableNotDisabledException re) { 577 expectedException = true; 578 } 579 assertFalse(expectedException); 580 HTableDescriptor modifiedHtd = new HTableDescriptor(this.admin.getTableDescriptor(tableName)); 581 assertFalse(htd.equals(modifiedHtd)); 582 assertTrue(copy.equals(modifiedHtd)); 583 assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize()); 584 assertEquals(key, modifiedHtd.getValue(key)); 585 586 // Now work on column family changes. 587 int countOfFamilies = modifiedHtd.getFamilies().size(); 588 assertTrue(countOfFamilies > 0); 589 HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next(); 590 int maxversions = hcd.getMaxVersions(); 591 final int newMaxVersions = maxversions + 1; 592 hcd.setMaxVersions(newMaxVersions); 593 final byte [] hcdName = hcd.getName(); 594 expectedException = false; 595 try { 596 this.admin.modifyColumnFamily(tableName, hcd); 597 } catch (TableNotDisabledException re) { 598 expectedException = true; 599 } 600 assertFalse(expectedException); 601 modifiedHtd = this.admin.getTableDescriptor(tableName); 602 HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName); 603 assertEquals(newMaxVersions, modifiedHcd.getMaxVersions()); 604 605 // Try adding a column 606 assertFalse(this.admin.isTableDisabled(tableName)); 607 final String xtracolName = "xtracol"; 608 HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName); 609 xtracol.setValue(xtracolName, xtracolName); 610 expectedException = false; 611 try { 612 this.admin.addColumnFamily(tableName, xtracol); 613 } catch (TableNotDisabledException re) { 614 expectedException = true; 615 } 616 // Add column should work even if the table is enabled 617 assertFalse(expectedException); 618 modifiedHtd = this.admin.getTableDescriptor(tableName); 619 hcd = modifiedHtd.getFamily(xtracol.getName()); 620 assertTrue(hcd != null); 621 assertTrue(hcd.getValue(xtracolName).equals(xtracolName)); 622 623 // Delete the just-added column. 624 this.admin.deleteColumnFamily(tableName, xtracol.getName()); 625 modifiedHtd = this.admin.getTableDescriptor(tableName); 626 hcd = modifiedHtd.getFamily(xtracol.getName()); 627 assertTrue(hcd == null); 628 629 // Delete the table 630 this.admin.disableTable(tableName); 631 this.admin.deleteTable(tableName); 632 this.admin.listTables(); 633 assertFalse(this.admin.tableExists(tableName)); 634 } 635 636 private void verifyRoundRobinDistribution(RegionLocator regionLocator, int expectedRegions) 637 throws IOException { 638 int numRS = TEST_UTIL.getMiniHBaseCluster().getNumLiveRegionServers(); 639 List<HRegionLocation> regions = regionLocator.getAllRegionLocations(); 640 Map<ServerName, List<RegionInfo>> server2Regions = new HashMap<>(); 641 for (HRegionLocation loc : regions) { 642 ServerName server = loc.getServerName(); 643 List<RegionInfo> regs = server2Regions.get(server); 644 if (regs == null) { 645 regs = new ArrayList<>(); 646 server2Regions.put(server, regs); 647 } 648 regs.add(loc.getRegionInfo()); 649 } 650 boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration()); 651 if (tablesOnMaster) { 652 // Ignore the master region server, 653 // which contains less regions by intention. 654 numRS--; 655 } 656 float average = (float) expectedRegions/numRS; 657 int min = (int)Math.floor(average); 658 int max = (int)Math.ceil(average); 659 for (List<RegionInfo> regionList : server2Regions.values()) { 660 assertTrue("numRS=" + numRS + ", min=" + min + ", max=" + max + 661 ", size=" + regionList.size() + ", tablesOnMaster=" + tablesOnMaster, 662 regionList.size() == min || regionList.size() == max); 663 } 664 } 665 666 @Test 667 public void testCreateTableNumberOfRegions() throws IOException, InterruptedException { 668 final TableName tableName = TableName.valueOf(name.getMethodName()); 669 HTableDescriptor desc = new HTableDescriptor(tableName); 670 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); 671 admin.createTable(desc); 672 List<HRegionLocation> regions; 673 try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) { 674 regions = l.getAllRegionLocations(); 675 assertEquals("Table should have only 1 region", 1, regions.size()); 676 } 677 678 TableName TABLE_2 = TableName.valueOf(tableName.getNameAsString() + "_2"); 679 desc = new HTableDescriptor(TABLE_2); 680 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); 681 admin.createTable(desc, new byte[][]{new byte[]{42}}); 682 try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(TABLE_2)) { 683 regions = l.getAllRegionLocations(); 684 assertEquals("Table should have only 2 region", 2, regions.size()); 685 } 686 687 TableName TABLE_3 = TableName.valueOf(tableName.getNameAsString() + "_3"); 688 desc = new HTableDescriptor(TABLE_3); 689 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); 690 admin.createTable(desc, "a".getBytes(), "z".getBytes(), 3); 691 try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(TABLE_3)) { 692 regions = l.getAllRegionLocations(); 693 assertEquals("Table should have only 3 region", 3, regions.size()); 694 } 695 696 TableName TABLE_4 = TableName.valueOf(tableName.getNameAsString() + "_4"); 697 desc = new HTableDescriptor(TABLE_4); 698 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); 699 try { 700 admin.createTable(desc, "a".getBytes(), "z".getBytes(), 2); 701 fail("Should not be able to create a table with only 2 regions using this API."); 702 } catch (IllegalArgumentException eae) { 703 // Expected 704 } 705 706 TableName TABLE_5 = TableName.valueOf(tableName.getNameAsString() + "_5"); 707 desc = new HTableDescriptor(TABLE_5); 708 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); 709 admin.createTable(desc, new byte[] { 1 }, new byte[] { 127 }, 16); 710 try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(TABLE_5)) { 711 regions = l.getAllRegionLocations(); 712 assertEquals("Table should have 16 region", 16, regions.size()); 713 } 714 } 715 716 @Test 717 public void testCreateTableWithRegions() throws IOException, InterruptedException { 718 final TableName tableName = TableName.valueOf(name.getMethodName()); 719 720 byte [][] splitKeys = { 721 new byte [] { 1, 1, 1 }, 722 new byte [] { 2, 2, 2 }, 723 new byte [] { 3, 3, 3 }, 724 new byte [] { 4, 4, 4 }, 725 new byte [] { 5, 5, 5 }, 726 new byte [] { 6, 6, 6 }, 727 new byte [] { 7, 7, 7 }, 728 new byte [] { 8, 8, 8 }, 729 new byte [] { 9, 9, 9 }, 730 }; 731 int expectedRegions = splitKeys.length + 1; 732 733 HTableDescriptor desc = new HTableDescriptor(tableName); 734 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); 735 admin.createTable(desc, splitKeys); 736 737 boolean tableAvailable = admin.isTableAvailable(tableName, splitKeys); 738 assertTrue("Table should be created with splitKyes + 1 rows in META", tableAvailable); 739 740 List<HRegionLocation> regions; 741 Iterator<HRegionLocation> hris; 742 RegionInfo hri; 743 ClusterConnection conn = (ClusterConnection) TEST_UTIL.getConnection(); 744 try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) { 745 regions = l.getAllRegionLocations(); 746 747 assertEquals("Tried to create " + expectedRegions + " regions " + 748 "but only found " + regions.size(), expectedRegions, regions.size()); 749 System.err.println("Found " + regions.size() + " regions"); 750 751 hris = regions.iterator(); 752 hri = hris.next().getRegionInfo(); 753 assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0); 754 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[0])); 755 hri = hris.next().getRegionInfo(); 756 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[0])); 757 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[1])); 758 hri = hris.next().getRegionInfo(); 759 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[1])); 760 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[2])); 761 hri = hris.next().getRegionInfo(); 762 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[2])); 763 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[3])); 764 hri = hris.next().getRegionInfo(); 765 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[3])); 766 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[4])); 767 hri = hris.next().getRegionInfo(); 768 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[4])); 769 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[5])); 770 hri = hris.next().getRegionInfo(); 771 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[5])); 772 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[6])); 773 hri = hris.next().getRegionInfo(); 774 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[6])); 775 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[7])); 776 hri = hris.next().getRegionInfo(); 777 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[7])); 778 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[8])); 779 hri = hris.next().getRegionInfo(); 780 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8])); 781 assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0); 782 783 verifyRoundRobinDistribution(l, expectedRegions); 784 } 785 786 787 788 // Now test using start/end with a number of regions 789 790 // Use 80 bit numbers to make sure we aren't limited 791 byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; 792 byte [] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; 793 794 // Splitting into 10 regions, we expect (null,1) ... (9, null) 795 // with (1,2) (2,3) (3,4) (4,5) (5,6) (6,7) (7,8) (8,9) in the middle 796 797 expectedRegions = 10; 798 799 TableName TABLE_2 = TableName.valueOf(tableName.getNameAsString() + "_2"); 800 801 desc = new HTableDescriptor(TABLE_2); 802 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); 803 admin = TEST_UTIL.getAdmin(); 804 admin.createTable(desc, startKey, endKey, expectedRegions); 805 806 try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(TABLE_2)) { 807 regions = l.getAllRegionLocations(); 808 assertEquals("Tried to create " + expectedRegions + " regions " + 809 "but only found " + regions.size(), expectedRegions, regions.size()); 810 System.err.println("Found " + regions.size() + " regions"); 811 812 hris = regions.iterator(); 813 hri = hris.next().getRegionInfo(); 814 assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0); 815 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 })); 816 hri = hris.next().getRegionInfo(); 817 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 })); 818 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 })); 819 hri = hris.next().getRegionInfo(); 820 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 })); 821 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 })); 822 hri = hris.next().getRegionInfo(); 823 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 })); 824 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 })); 825 hri = hris.next().getRegionInfo(); 826 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 })); 827 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 })); 828 hri = hris.next().getRegionInfo(); 829 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 })); 830 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 })); 831 hri = hris.next().getRegionInfo(); 832 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 })); 833 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 })); 834 hri = hris.next().getRegionInfo(); 835 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 })); 836 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 })); 837 hri = hris.next().getRegionInfo(); 838 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 })); 839 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 })); 840 hri = hris.next().getRegionInfo(); 841 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 })); 842 assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0); 843 844 verifyRoundRobinDistribution(l, expectedRegions); 845 } 846 847 // Try once more with something that divides into something infinite 848 849 startKey = new byte [] { 0, 0, 0, 0, 0, 0 }; 850 endKey = new byte [] { 1, 0, 0, 0, 0, 0 }; 851 852 expectedRegions = 5; 853 854 TableName TABLE_3 = TableName.valueOf(tableName.getNameAsString() + "_3"); 855 856 desc = new HTableDescriptor(TABLE_3); 857 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); 858 admin = TEST_UTIL.getAdmin(); 859 admin.createTable(desc, startKey, endKey, expectedRegions); 860 861 862 try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(TABLE_3)) { 863 regions = l.getAllRegionLocations(); 864 assertEquals("Tried to create " + expectedRegions + " regions " + 865 "but only found " + regions.size(), expectedRegions, regions.size()); 866 System.err.println("Found " + regions.size() + " regions"); 867 868 verifyRoundRobinDistribution(l, expectedRegions); 869 } 870 871 872 // Try an invalid case where there are duplicate split keys 873 splitKeys = new byte [][] { 874 new byte [] { 1, 1, 1 }, 875 new byte [] { 2, 2, 2 }, 876 new byte [] { 3, 3, 3 }, 877 new byte [] { 2, 2, 2 } 878 }; 879 880 TableName TABLE_4 = TableName.valueOf(tableName.getNameAsString() + "_4"); 881 desc = new HTableDescriptor(TABLE_4); 882 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); 883 try { 884 admin.createTable(desc, splitKeys); 885 assertTrue("Should not be able to create this table because of " + 886 "duplicate split keys", false); 887 } catch(IllegalArgumentException iae) { 888 // Expected 889 } 890 } 891 892 @Test 893 public void testTableAvailableWithRandomSplitKeys() throws Exception { 894 final TableName tableName = TableName.valueOf(name.getMethodName()); 895 HTableDescriptor desc = new HTableDescriptor(tableName); 896 desc.addFamily(new HColumnDescriptor("col")); 897 byte[][] splitKeys = new byte[1][]; 898 splitKeys = new byte [][] { 899 new byte [] { 1, 1, 1 }, 900 new byte [] { 2, 2, 2 } 901 }; 902 admin.createTable(desc); 903 boolean tableAvailable = admin.isTableAvailable(tableName, splitKeys); 904 assertFalse("Table should be created with 1 row in META", tableAvailable); 905 } 906 907 @Test 908 public void testCreateTableWithOnlyEmptyStartRow() throws IOException { 909 final byte[] tableName = Bytes.toBytes(name.getMethodName()); 910 byte[][] splitKeys = new byte[1][]; 911 splitKeys[0] = HConstants.EMPTY_BYTE_ARRAY; 912 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); 913 desc.addFamily(new HColumnDescriptor("col")); 914 try { 915 admin.createTable(desc, splitKeys); 916 fail("Test case should fail as empty split key is passed."); 917 } catch (IllegalArgumentException e) { 918 } 919 } 920 921 @Test 922 public void testCreateTableWithEmptyRowInTheSplitKeys() throws IOException{ 923 final byte[] tableName = Bytes.toBytes(name.getMethodName()); 924 byte[][] splitKeys = new byte[3][]; 925 splitKeys[0] = "region1".getBytes(); 926 splitKeys[1] = HConstants.EMPTY_BYTE_ARRAY; 927 splitKeys[2] = "region2".getBytes(); 928 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); 929 desc.addFamily(new HColumnDescriptor("col")); 930 try { 931 admin.createTable(desc, splitKeys); 932 fail("Test case should fail as empty split key is passed."); 933 } catch (IllegalArgumentException e) { 934 LOG.info("Expected ", e); 935 } 936 } 937 938 @Test 939 public void testTableExist() throws IOException { 940 final TableName table = TableName.valueOf(name.getMethodName()); 941 boolean exist; 942 exist = this.admin.tableExists(table); 943 assertEquals(false, exist); 944 TEST_UTIL.createTable(table, HConstants.CATALOG_FAMILY); 945 exist = this.admin.tableExists(table); 946 assertEquals(true, exist); 947 } 948 949 /** 950 * Tests forcing split from client and having scanners successfully ride over split. 951 * @throws Exception 952 * @throws IOException 953 */ 954 @Test 955 public void testForceSplit() throws Exception { 956 byte[][] familyNames = new byte[][] { Bytes.toBytes("cf") }; 957 int[] rowCounts = new int[] { 6000 }; 958 int numVersions = HColumnDescriptor.DEFAULT_VERSIONS; 959 int blockSize = 256; 960 splitTest(null, familyNames, rowCounts, numVersions, blockSize, true); 961 962 byte[] splitKey = Bytes.toBytes(3500); 963 splitTest(splitKey, familyNames, rowCounts, numVersions, blockSize, true); 964 // test regionSplitSync 965 splitTest(splitKey, familyNames, rowCounts, numVersions, blockSize, false); 966 } 967 968 /** 969 * Test retain assignment on enableTable. 970 * 971 * @throws IOException 972 */ 973 @Test 974 public void testEnableTableRetainAssignment() throws IOException { 975 final TableName tableName = TableName.valueOf(name.getMethodName()); 976 byte[][] splitKeys = { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 }, 977 new byte[] { 3, 3, 3 }, new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 }, 978 new byte[] { 6, 6, 6 }, new byte[] { 7, 7, 7 }, new byte[] { 8, 8, 8 }, 979 new byte[] { 9, 9, 9 } }; 980 int expectedRegions = splitKeys.length + 1; 981 HTableDescriptor desc = new HTableDescriptor(tableName); 982 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); 983 admin.createTable(desc, splitKeys); 984 985 try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) { 986 List<HRegionLocation> regions = l.getAllRegionLocations(); 987 988 assertEquals( 989 "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), 990 expectedRegions, regions.size()); 991 // Disable table. 992 admin.disableTable(tableName); 993 // Enable table, use retain assignment to assign regions. 994 admin.enableTable(tableName); 995 List<HRegionLocation> regions2 = l.getAllRegionLocations(); 996 997 // Check the assignment. 998 assertEquals(regions.size(), regions2.size()); 999 assertTrue(regions2.containsAll(regions)); 1000 } 1001 } 1002 1003 /** 1004 * Multi-family scenario. Tests forcing split from client and 1005 * having scanners successfully ride over split. 1006 * @throws Exception 1007 * @throws IOException 1008 */ 1009 @Test 1010 public void testForceSplitMultiFamily() throws Exception { 1011 int numVersions = HColumnDescriptor.DEFAULT_VERSIONS; 1012 1013 // use small HFile block size so that we can have lots of blocks in HFile 1014 // Otherwise, if there is only one block, 1015 // HFileBlockIndex.midKey()'s value == startKey 1016 int blockSize = 256; 1017 byte[][] familyNames = new byte[][] { Bytes.toBytes("cf1"), 1018 Bytes.toBytes("cf2") }; 1019 1020 // one of the column families isn't splittable 1021 int[] rowCounts = new int[] { 6000, 1 }; 1022 splitTest(null, familyNames, rowCounts, numVersions, blockSize, true); 1023 1024 rowCounts = new int[] { 1, 6000 }; 1025 splitTest(null, familyNames, rowCounts, numVersions, blockSize, true); 1026 1027 // one column family has much smaller data than the other 1028 // the split key should be based on the largest column family 1029 rowCounts = new int[] { 6000, 300 }; 1030 splitTest(null, familyNames, rowCounts, numVersions, blockSize, true); 1031 1032 rowCounts = new int[] { 300, 6000 }; 1033 splitTest(null, familyNames, rowCounts, numVersions, blockSize, true); 1034 1035 } 1036 1037 void splitTest(byte[] splitPoint, byte[][] familyNames, int[] rowCounts, 1038 int numVersions, int blockSize, boolean async) throws Exception { 1039 TableName tableName = TableName.valueOf("testForceSplit"); 1040 StringBuilder sb = new StringBuilder(); 1041 // Add tail to String so can see better in logs where a test is running. 1042 for (int i = 0; i < rowCounts.length; i++) { 1043 sb.append("_").append(Integer.toString(rowCounts[i])); 1044 } 1045 assertFalse(admin.tableExists(tableName)); 1046 try (final Table table = TEST_UTIL.createTable(tableName, familyNames, 1047 numVersions, blockSize); 1048 final RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { 1049 1050 int rowCount = 0; 1051 byte[] q = new byte[0]; 1052 1053 // insert rows into column families. The number of rows that have values 1054 // in a specific column family is decided by rowCounts[familyIndex] 1055 for (int index = 0; index < familyNames.length; index++) { 1056 ArrayList<Put> puts = new ArrayList<>(rowCounts[index]); 1057 for (int i = 0; i < rowCounts[index]; i++) { 1058 byte[] k = Bytes.toBytes(i); 1059 Put put = new Put(k); 1060 put.addColumn(familyNames[index], q, k); 1061 puts.add(put); 1062 } 1063 table.put(puts); 1064 1065 if (rowCount < rowCounts[index]) { 1066 rowCount = rowCounts[index]; 1067 } 1068 } 1069 1070 // get the initial layout (should just be one region) 1071 List<HRegionLocation> m = locator.getAllRegionLocations(); 1072 LOG.info("Initial regions (" + m.size() + "): " + m); 1073 assertTrue(m.size() == 1); 1074 1075 // Verify row count 1076 Scan scan = new Scan(); 1077 ResultScanner scanner = table.getScanner(scan); 1078 int rows = 0; 1079 for (@SuppressWarnings("unused") Result result : scanner) { 1080 rows++; 1081 } 1082 scanner.close(); 1083 assertEquals(rowCount, rows); 1084 1085 // Have an outstanding scan going on to make sure we can scan over splits. 1086 scan = new Scan(); 1087 scanner = table.getScanner(scan); 1088 // Scan first row so we are into first region before split happens. 1089 scanner.next(); 1090 1091 // Split the table 1092 if (async) { 1093 this.admin.split(tableName, splitPoint); 1094 final AtomicInteger count = new AtomicInteger(0); 1095 Thread t = new Thread("CheckForSplit") { 1096 @Override public void run() { 1097 for (int i = 0; i < 45; i++) { 1098 try { 1099 sleep(1000); 1100 } catch (InterruptedException e) { 1101 continue; 1102 } 1103 // check again 1104 List<HRegionLocation> regions = null; 1105 try { 1106 regions = locator.getAllRegionLocations(); 1107 } catch (IOException e) { 1108 e.printStackTrace(); 1109 } 1110 if (regions == null) continue; 1111 count.set(regions.size()); 1112 if (count.get() >= 2) { 1113 LOG.info("Found: " + regions); 1114 break; 1115 } 1116 LOG.debug("Cycle waiting on split"); 1117 } 1118 LOG.debug("CheckForSplit thread exited, current region count: " + count.get()); 1119 } 1120 }; 1121 t.setPriority(Thread.NORM_PRIORITY - 2); 1122 t.start(); 1123 t.join(); 1124 } else { 1125 // Sync split region, no need to create a thread to check 1126 ((HBaseAdmin)admin).splitRegionSync(m.get(0).getRegionInfo().getRegionName(), splitPoint); 1127 } 1128 1129 // Verify row count 1130 rows = 1; // We counted one row above. 1131 for (@SuppressWarnings("unused") Result result : scanner) { 1132 rows++; 1133 if (rows > rowCount) { 1134 scanner.close(); 1135 assertTrue("Scanned more than expected (" + rowCount + ")", false); 1136 } 1137 } 1138 scanner.close(); 1139 assertEquals(rowCount, rows); 1140 1141 List<HRegionLocation> regions = null; 1142 try { 1143 regions = locator.getAllRegionLocations(); 1144 } catch (IOException e) { 1145 e.printStackTrace(); 1146 } 1147 assertEquals(2, regions.size()); 1148 if (splitPoint != null) { 1149 // make sure the split point matches our explicit configuration 1150 assertEquals(Bytes.toString(splitPoint), 1151 Bytes.toString(regions.get(0).getRegionInfo().getEndKey())); 1152 assertEquals(Bytes.toString(splitPoint), 1153 Bytes.toString(regions.get(1).getRegionInfo().getStartKey())); 1154 LOG.debug("Properly split on " + Bytes.toString(splitPoint)); 1155 } else { 1156 if (familyNames.length > 1) { 1157 int splitKey = Bytes.toInt(regions.get(0).getRegionInfo().getEndKey()); 1158 // check if splitKey is based on the largest column family 1159 // in terms of it store size 1160 int deltaForLargestFamily = Math.abs(rowCount / 2 - splitKey); 1161 LOG.debug("SplitKey=" + splitKey + "&deltaForLargestFamily=" + deltaForLargestFamily + 1162 ", r=" + regions.get(0).getRegionInfo()); 1163 for (int index = 0; index < familyNames.length; index++) { 1164 int delta = Math.abs(rowCounts[index] / 2 - splitKey); 1165 if (delta < deltaForLargestFamily) { 1166 assertTrue("Delta " + delta + " for family " + index + " should be at least " 1167 + "deltaForLargestFamily " + deltaForLargestFamily, false); 1168 } 1169 } 1170 } 1171 } 1172 TEST_UTIL.deleteTable(tableName); 1173 } 1174 } 1175 1176 @Test 1177 public void testSplitAndMergeWithReplicaTable() throws Exception { 1178 // The test tries to directly split replica regions and directly merge replica regions. These 1179 // are not allowed. The test validates that. Then the test does a valid split/merge of allowed 1180 // regions. 1181 // Set up a table with 3 regions and replication set to 3 1182 final TableName tableName = TableName.valueOf(name.getMethodName()); 1183 HTableDescriptor desc = new HTableDescriptor(tableName); 1184 desc.setRegionReplication(3); 1185 byte[] cf = "f".getBytes(); 1186 HColumnDescriptor hcd = new HColumnDescriptor(cf); 1187 desc.addFamily(hcd); 1188 byte[][] splitRows = new byte[2][]; 1189 splitRows[0] = new byte[]{(byte)'4'}; 1190 splitRows[1] = new byte[]{(byte)'7'}; 1191 TEST_UTIL.getAdmin().createTable(desc, splitRows); 1192 List<HRegion> oldRegions; 1193 do { 1194 oldRegions = TEST_UTIL.getHBaseCluster().getRegions(tableName); 1195 Thread.sleep(10); 1196 } while (oldRegions.size() != 9); //3 regions * 3 replicas 1197 // write some data to the table 1198 Table ht = TEST_UTIL.getConnection().getTable(tableName); 1199 List<Put> puts = new ArrayList<>(); 1200 byte[] qualifier = "c".getBytes(); 1201 Put put = new Put(new byte[]{(byte)'1'}); 1202 put.addColumn(cf, qualifier, "100".getBytes()); 1203 puts.add(put); 1204 put = new Put(new byte[]{(byte)'6'}); 1205 put.addColumn(cf, qualifier, "100".getBytes()); 1206 puts.add(put); 1207 put = new Put(new byte[]{(byte)'8'}); 1208 put.addColumn(cf, qualifier, "100".getBytes()); 1209 puts.add(put); 1210 ht.put(puts); 1211 ht.close(); 1212 List<Pair<RegionInfo, ServerName>> regions = 1213 MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getConnection(), tableName); 1214 boolean gotException = false; 1215 // the element at index 1 would be a replica (since the metareader gives us ordered 1216 // regions). Try splitting that region via the split API . Should fail 1217 try { 1218 TEST_UTIL.getAdmin().splitRegionAsync(regions.get(1).getFirst().getRegionName()).get(); 1219 } catch (IllegalArgumentException ex) { 1220 gotException = true; 1221 } 1222 assertTrue(gotException); 1223 gotException = false; 1224 // the element at index 1 would be a replica (since the metareader gives us ordered 1225 // regions). Try splitting that region via a different split API (the difference is 1226 // this API goes direct to the regionserver skipping any checks in the admin). Should fail 1227 try { 1228 TEST_UTIL.getHBaseAdmin().splitRegionAsync(regions.get(1).getFirst(), 1229 new byte[]{(byte)'1'}); 1230 } catch (IOException ex) { 1231 gotException = true; 1232 } 1233 assertTrue(gotException); 1234 1235 gotException = false; 1236 //testing Sync split operation 1237 try { 1238 TEST_UTIL.getHBaseAdmin().splitRegionSync(regions.get(1).getFirst().getRegionName(), 1239 new byte[]{(byte)'1'}); 1240 } catch (IllegalArgumentException ex) { 1241 gotException = true; 1242 } 1243 assertTrue(gotException); 1244 1245 gotException = false; 1246 // Try merging a replica with another. Should fail. 1247 try { 1248 TEST_UTIL.getHBaseAdmin().mergeRegionsSync( 1249 regions.get(1).getFirst().getEncodedNameAsBytes(), 1250 regions.get(2).getFirst().getEncodedNameAsBytes(), 1251 true); 1252 } catch (IllegalArgumentException m) { 1253 gotException = true; 1254 } 1255 assertTrue(gotException); 1256 // Try going to the master directly (that will skip the check in admin) 1257 try { 1258 byte[][] nameofRegionsToMerge = new byte[2][]; 1259 nameofRegionsToMerge[0] = regions.get(1).getFirst().getEncodedNameAsBytes(); 1260 nameofRegionsToMerge[1] = regions.get(2).getFirst().getEncodedNameAsBytes(); 1261 MergeTableRegionsRequest request = RequestConverter 1262 .buildMergeTableRegionsRequest( 1263 nameofRegionsToMerge, 1264 true, 1265 HConstants.NO_NONCE, 1266 HConstants.NO_NONCE); 1267 ((ClusterConnection) TEST_UTIL.getAdmin().getConnection()).getMaster() 1268 .mergeTableRegions(null, request); 1269 } catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException m) { 1270 Throwable t = m.getCause(); 1271 do { 1272 if (t instanceof MergeRegionException) { 1273 gotException = true; 1274 break; 1275 } 1276 t = t.getCause(); 1277 } while (t != null); 1278 } 1279 assertTrue(gotException); 1280 } 1281 1282 @Test (expected=IllegalArgumentException.class) 1283 public void testInvalidHColumnDescriptor() throws IOException { 1284 new HColumnDescriptor("/cfamily/name"); 1285 } 1286 1287 @Test 1288 public void testEnableDisableAddColumnDeleteColumn() throws Exception { 1289 final TableName tableName = TableName.valueOf(name.getMethodName()); 1290 TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); 1291 while (!this.admin.isTableEnabled(TableName.valueOf(name.getMethodName()))) { 1292 Thread.sleep(10); 1293 } 1294 this.admin.disableTable(tableName); 1295 try { 1296 TEST_UTIL.getConnection().getTable(tableName); 1297 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) { 1298 //expected 1299 } 1300 1301 this.admin.addColumnFamily(tableName, new HColumnDescriptor("col2")); 1302 this.admin.enableTable(tableName); 1303 try { 1304 this.admin.deleteColumnFamily(tableName, Bytes.toBytes("col2")); 1305 } catch (TableNotDisabledException e) { 1306 LOG.info(e.toString(), e); 1307 } 1308 this.admin.disableTable(tableName); 1309 this.admin.deleteTable(tableName); 1310 } 1311 1312 @Test 1313 public void testDeleteLastColumnFamily() throws Exception { 1314 final TableName tableName = TableName.valueOf(name.getMethodName()); 1315 TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); 1316 while (!this.admin.isTableEnabled(TableName.valueOf(name.getMethodName()))) { 1317 Thread.sleep(10); 1318 } 1319 1320 // test for enabled table 1321 try { 1322 this.admin.deleteColumnFamily(tableName, HConstants.CATALOG_FAMILY); 1323 fail("Should have failed to delete the only column family of a table"); 1324 } catch (InvalidFamilyOperationException ex) { 1325 // expected 1326 } 1327 1328 // test for disabled table 1329 this.admin.disableTable(tableName); 1330 1331 try { 1332 this.admin.deleteColumnFamily(tableName, HConstants.CATALOG_FAMILY); 1333 fail("Should have failed to delete the only column family of a table"); 1334 } catch (InvalidFamilyOperationException ex) { 1335 // expected 1336 } 1337 1338 this.admin.deleteTable(tableName); 1339 } 1340 1341 /* 1342 * Test DFS replication for column families, where one CF has default replication(3) and the other 1343 * is set to 1. 1344 */ 1345 @Test 1346 public void testHFileReplication() throws Exception { 1347 final TableName tableName = TableName.valueOf(this.name.getMethodName()); 1348 String fn1 = "rep1"; 1349 HColumnDescriptor hcd1 = new HColumnDescriptor(fn1); 1350 hcd1.setDFSReplication((short) 1); 1351 String fn = "defaultRep"; 1352 HColumnDescriptor hcd = new HColumnDescriptor(fn); 1353 HTableDescriptor htd = new HTableDescriptor(tableName); 1354 htd.addFamily(hcd); 1355 htd.addFamily(hcd1); 1356 Table table = TEST_UTIL.createTable(htd, null); 1357 TEST_UTIL.waitTableAvailable(tableName); 1358 Put p = new Put(Bytes.toBytes("defaultRep_rk")); 1359 byte[] q1 = Bytes.toBytes("q1"); 1360 byte[] v1 = Bytes.toBytes("v1"); 1361 p.addColumn(Bytes.toBytes(fn), q1, v1); 1362 List<Put> puts = new ArrayList<>(2); 1363 puts.add(p); 1364 p = new Put(Bytes.toBytes("rep1_rk")); 1365 p.addColumn(Bytes.toBytes(fn1), q1, v1); 1366 puts.add(p); 1367 try { 1368 table.put(puts); 1369 admin.flush(tableName); 1370 1371 List<HRegion> regions = TEST_UTIL.getMiniHBaseCluster().getRegions(tableName); 1372 for (HRegion r : regions) { 1373 HStore store = r.getStore(Bytes.toBytes(fn)); 1374 for (HStoreFile sf : store.getStorefiles()) { 1375 assertTrue(sf.toString().contains(fn)); 1376 assertTrue("Column family " + fn + " should have 3 copies", 1377 FSUtils.getDefaultReplication(TEST_UTIL.getTestFileSystem(), sf.getPath()) == (sf 1378 .getFileInfo().getFileStatus().getReplication())); 1379 } 1380 1381 store = r.getStore(Bytes.toBytes(fn1)); 1382 for (HStoreFile sf : store.getStorefiles()) { 1383 assertTrue(sf.toString().contains(fn1)); 1384 assertTrue("Column family " + fn1 + " should have only 1 copy", 1 == sf.getFileInfo() 1385 .getFileStatus().getReplication()); 1386 } 1387 } 1388 } finally { 1389 if (admin.isTableEnabled(tableName)) { 1390 this.admin.disableTable(tableName); 1391 this.admin.deleteTable(tableName); 1392 } 1393 } 1394 } 1395 1396 @Test 1397 public void testMergeRegions() throws Exception { 1398 final TableName tableName = TableName.valueOf(name.getMethodName()); 1399 HColumnDescriptor cd = new HColumnDescriptor("d"); 1400 HTableDescriptor td = new HTableDescriptor(tableName); 1401 td.addFamily(cd); 1402 byte[][] splitRows = new byte[2][]; 1403 splitRows[0] = new byte[]{(byte)'3'}; 1404 splitRows[1] = new byte[]{(byte)'6'}; 1405 try { 1406 TEST_UTIL.createTable(td, splitRows); 1407 TEST_UTIL.waitTableAvailable(tableName); 1408 1409 List<RegionInfo> tableRegions; 1410 RegionInfo regionA; 1411 RegionInfo regionB; 1412 1413 // merge with full name 1414 tableRegions = admin.getRegions(tableName); 1415 assertEquals(3, admin.getTableRegions(tableName).size()); 1416 regionA = tableRegions.get(0); 1417 regionB = tableRegions.get(1); 1418 // TODO convert this to version that is synchronous (See HBASE-16668) 1419 admin.mergeRegionsAsync(regionA.getRegionName(), regionB.getRegionName(), false) 1420 .get(60, TimeUnit.SECONDS); 1421 1422 assertEquals(2, admin.getTableRegions(tableName).size()); 1423 1424 // merge with encoded name 1425 tableRegions = admin.getRegions(tableName); 1426 regionA = tableRegions.get(0); 1427 regionB = tableRegions.get(1); 1428 // TODO convert this to version that is synchronous (See HBASE-16668) 1429 admin.mergeRegionsAsync( 1430 regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false) 1431 .get(60, TimeUnit.SECONDS); 1432 1433 assertEquals(1, admin.getTableRegions(tableName).size()); 1434 } finally { 1435 this.admin.disableTable(tableName); 1436 this.admin.deleteTable(tableName); 1437 } 1438 } 1439 1440 @Test 1441 public void testMergeRegionsInvalidRegionCount() 1442 throws IOException, InterruptedException, ExecutionException { 1443 TableName tableName = TableName.valueOf(name.getMethodName()); 1444 TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName) 1445 .setColumnFamily(ColumnFamilyDescriptorBuilder.of("d")).build(); 1446 byte[][] splitRows = new byte[2][]; 1447 splitRows[0] = new byte[] { (byte) '3' }; 1448 splitRows[1] = new byte[] { (byte) '6' }; 1449 try { 1450 TEST_UTIL.createTable(td, splitRows); 1451 TEST_UTIL.waitTableAvailable(tableName); 1452 1453 List<RegionInfo> tableRegions = admin.getRegions(tableName); 1454 // 0 1455 try { 1456 admin.mergeRegionsAsync(new byte[0][0], false).get(); 1457 fail(); 1458 } catch (IllegalArgumentException e) { 1459 // expected 1460 } 1461 // 1 1462 try { 1463 admin.mergeRegionsAsync(new byte[][] { tableRegions.get(0).getEncodedNameAsBytes() }, false) 1464 .get(); 1465 fail(); 1466 } catch (IllegalArgumentException e) { 1467 // expected 1468 } 1469 } finally { 1470 admin.disableTable(tableName); 1471 admin.deleteTable(tableName); 1472 } 1473 } 1474 1475 @Test 1476 public void testSplitShouldNotHappenIfSplitIsDisabledForTable() 1477 throws Exception { 1478 final TableName tableName = TableName.valueOf(name.getMethodName()); 1479 HTableDescriptor htd = new HTableDescriptor(tableName); 1480 htd.addFamily(new HColumnDescriptor("f")); 1481 htd.setRegionSplitPolicyClassName(DisabledRegionSplitPolicy.class.getName()); 1482 Table table = TEST_UTIL.createTable(htd, null); 1483 for(int i = 0; i < 10; i++) { 1484 Put p = new Put(Bytes.toBytes("row"+i)); 1485 byte[] q1 = Bytes.toBytes("q1"); 1486 byte[] v1 = Bytes.toBytes("v1"); 1487 p.addColumn(Bytes.toBytes("f"), q1, v1); 1488 table.put(p); 1489 } 1490 this.admin.flush(tableName); 1491 try { 1492 this.admin.split(tableName, Bytes.toBytes("row5")); 1493 Threads.sleep(10000); 1494 } catch (Exception e) { 1495 // Nothing to do. 1496 } 1497 // Split should not happen. 1498 List<RegionInfo> allRegions = MetaTableAccessor.getTableRegions( 1499 this.admin.getConnection(), tableName, true); 1500 assertEquals(1, allRegions.size()); 1501 } 1502 1503 @Test 1504 public void testCloneTableSchema() throws Exception { 1505 final TableName tableName = TableName.valueOf(name.getMethodName()); 1506 final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); 1507 testCloneTableSchema(tableName, newTableName, false); 1508 } 1509 1510 @Test 1511 public void testCloneTableSchemaPreservingSplits() throws Exception { 1512 final TableName tableName = TableName.valueOf(name.getMethodName()); 1513 final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); 1514 testCloneTableSchema(tableName, newTableName, true); 1515 } 1516 1517 private void testCloneTableSchema(final TableName tableName, 1518 final TableName newTableName, boolean preserveSplits) throws Exception { 1519 byte[] FAMILY_0 = Bytes.toBytes("cf0"); 1520 byte[] FAMILY_1 = Bytes.toBytes("cf1"); 1521 byte[][] splitKeys = new byte[2][]; 1522 splitKeys[0] = Bytes.toBytes(4); 1523 splitKeys[1] = Bytes.toBytes(8); 1524 int NUM_FAMILYS = 2; 1525 int NUM_REGIONS = 3; 1526 int BLOCK_SIZE = 1024; 1527 int TTL = 86400; 1528 boolean BLOCK_CACHE = false; 1529 1530 // Create the table 1531 TableDescriptor tableDesc = TableDescriptorBuilder 1532 .newBuilder(tableName) 1533 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0)) 1534 .setColumnFamily(ColumnFamilyDescriptorBuilder 1535 .newBuilder(FAMILY_1) 1536 .setBlocksize(BLOCK_SIZE) 1537 .setBlockCacheEnabled(BLOCK_CACHE) 1538 .setTimeToLive(TTL) 1539 .build() 1540 ).build(); 1541 admin.createTable(tableDesc, splitKeys); 1542 1543 assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(tableName).size()); 1544 assertTrue("Table should be created with splitKyes + 1 rows in META", 1545 admin.isTableAvailable(tableName, splitKeys)); 1546 1547 // clone & Verify 1548 admin.cloneTableSchema(tableName, newTableName, preserveSplits); 1549 TableDescriptor newTableDesc = admin.getDescriptor(newTableName); 1550 1551 assertEquals(NUM_FAMILYS, newTableDesc.getColumnFamilyCount()); 1552 assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize()); 1553 assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled()); 1554 assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive()); 1555 TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc); 1556 1557 if (preserveSplits) { 1558 assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size()); 1559 assertTrue("New table should be created with splitKyes + 1 rows in META", 1560 admin.isTableAvailable(newTableName, splitKeys)); 1561 } else { 1562 assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size()); 1563 } 1564 } 1565 1566 @Test 1567 public void testCloneTableSchemaWithNonExistentSourceTable() throws Exception { 1568 final TableName tableName = TableName.valueOf(name.getMethodName()); 1569 final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); 1570 // test for non-existent source table 1571 try { 1572 admin.cloneTableSchema(tableName, newTableName, false); 1573 fail("Should have failed to create a new table by cloning non-existent source table."); 1574 } catch (TableNotFoundException ex) { 1575 // expected 1576 } 1577 } 1578 1579 @Test 1580 public void testCloneTableSchemaWithExistentDestinationTable() throws Exception { 1581 final TableName tableName = TableName.valueOf(name.getMethodName()); 1582 final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new"); 1583 byte[] FAMILY_0 = Bytes.toBytes("cf0"); 1584 TEST_UTIL.createTable(tableName, FAMILY_0); 1585 TEST_UTIL.createTable(newTableName, FAMILY_0); 1586 // test for existent destination table 1587 try { 1588 admin.cloneTableSchema(tableName, newTableName, false); 1589 fail("Should have failed to create a existent table."); 1590 } catch (TableExistsException ex) { 1591 // expected 1592 } 1593 } 1594 1595 @Test 1596 public void testModifyTableOnTableWithRegionReplicas() throws Exception { 1597 TableName tableName = TableName.valueOf(name.getMethodName()); 1598 TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) 1599 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf"))) 1600 .setRegionReplication(5) 1601 .build(); 1602 1603 admin.createTable(desc); 1604 1605 int maxFileSize = 10000000; 1606 TableDescriptor newDesc = TableDescriptorBuilder.newBuilder(desc) 1607 .setMaxFileSize(maxFileSize) 1608 .build(); 1609 1610 admin.modifyTable(newDesc); 1611 TableDescriptor newTableDesc = admin.getDescriptor(tableName); 1612 assertEquals(maxFileSize, newTableDesc.getMaxFileSize()); 1613 } 1614}