Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
021import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertFalse;
024import static org.junit.Assert.assertTrue;
025import static org.junit.Assert.fail;
026
027import java.util.Iterator;
028import java.util.List;
029import java.util.Optional;
030import java.util.concurrent.CompletionException;
031import org.apache.hadoop.hbase.ClientMetaTableAccessor;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.HRegionLocation;
035import org.apache.hadoop.hbase.TableExistsException;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.TableNotFoundException;
038import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
039import org.apache.hadoop.hbase.testclassification.ClientTests;
040import org.apache.hadoop.hbase.testclassification.LargeTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.junit.ClassRule;
043import org.junit.Test;
044import org.junit.experimental.categories.Category;
045import org.junit.runner.RunWith;
046import org.junit.runners.Parameterized;
047
048/**
049 * Class to test asynchronous table admin operations.
050 * @see TestAsyncTableAdminApi2 This test and it used to be joined it was taking longer than our ten
051 *      minute timeout so they were split.
052 * @see TestAsyncTableAdminApi3 Another split out from this class so each runs under ten minutes.
053 */
054@RunWith(Parameterized.class)
055@Category({ LargeTests.class, ClientTests.class })
056public class TestAsyncTableAdminApi extends TestAsyncAdminBase {
057
058  @ClassRule
059  public static final HBaseClassTestRule CLASS_RULE =
060    HBaseClassTestRule.forClass(TestAsyncTableAdminApi.class);
061
062  @Test
063  public void testCreateTable() throws Exception {
064    List<TableDescriptor> tables = admin.listTableDescriptors().get();
065    int numTables = tables.size();
066    createTableWithDefaultConf(tableName);
067    tables = admin.listTableDescriptors().get();
068    assertEquals(numTables + 1, tables.size());
069    assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster().getMaster()
070      .getTableStateManager().isTableState(tableName, TableState.State.ENABLED));
071    assertEquals(TableState.State.ENABLED, getStateFromMeta(tableName));
072  }
073
074  static TableState.State getStateFromMeta(TableName table) throws Exception {
075    Optional<TableState> state = ClientMetaTableAccessor
076      .getTableState(ASYNC_CONN.getTable(TableName.META_TABLE_NAME), table).get();
077    assertTrue(state.isPresent());
078    return state.get().getState();
079  }
080
081  @Test
082  public void testCreateTableNumberOfRegions() throws Exception {
083    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
084
085    createTableWithDefaultConf(tableName);
086    List<HRegionLocation> regionLocations =
087      ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
088    assertEquals("Table should have only 1 region", 1, regionLocations.size());
089
090    final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "_2");
091    createTableWithDefaultConf(tableName2, new byte[][] { new byte[] { 42 } });
092    regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName2).get();
093    assertEquals("Table should have only 2 region", 2, regionLocations.size());
094
095    final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "_3");
096    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName3);
097    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
098    admin.createTable(builder.build(), Bytes.toBytes("a"), Bytes.toBytes("z"), 3).join();
099    regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName3).get();
100    assertEquals("Table should have only 3 region", 3, regionLocations.size());
101
102    final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "_4");
103    builder = TableDescriptorBuilder.newBuilder(tableName4);
104    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
105    try {
106      admin.createTable(builder.build(), Bytes.toBytes("a"), Bytes.toBytes("z"), 2).join();
107      fail("Should not be able to create a table with only 2 regions using this API.");
108    } catch (CompletionException e) {
109      assertTrue(e.getCause() instanceof IllegalArgumentException);
110    }
111
112    final TableName tableName5 = TableName.valueOf(tableName.getNameAsString() + "_5");
113    builder = TableDescriptorBuilder.newBuilder(tableName5);
114    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
115    admin.createTable(builder.build(), new byte[] { 1 }, new byte[] { 127 }, 16).join();
116    regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName5).get();
117    assertEquals("Table should have 16 region", 16, regionLocations.size());
118  }
119
120  @Test
121  public void testCreateTableWithRegions() throws Exception {
122    byte[][] splitKeys = { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 }, new byte[] { 3, 3, 3 },
123      new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 }, new byte[] { 6, 6, 6 },
124      new byte[] { 7, 7, 7 }, new byte[] { 8, 8, 8 }, new byte[] { 9, 9, 9 }, };
125    int expectedRegions = splitKeys.length + 1;
126    createTableWithDefaultConf(tableName, splitKeys);
127
128    boolean tableAvailable = admin.isTableAvailable(tableName).get();
129    assertTrue("Table should be created with splitKyes + 1 rows in META", tableAvailable);
130
131    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
132    List<HRegionLocation> regions =
133      ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
134    Iterator<HRegionLocation> hris = regions.iterator();
135
136    assertEquals(
137      "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
138      expectedRegions, regions.size());
139    System.err.println("Found " + regions.size() + " regions");
140
141    RegionInfo hri;
142    hris = regions.iterator();
143    hri = hris.next().getRegion();
144    assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
145    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[0]));
146    hri = hris.next().getRegion();
147    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[0]));
148    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[1]));
149    hri = hris.next().getRegion();
150    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[1]));
151    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[2]));
152    hri = hris.next().getRegion();
153    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[2]));
154    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[3]));
155    hri = hris.next().getRegion();
156    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[3]));
157    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[4]));
158    hri = hris.next().getRegion();
159    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[4]));
160    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[5]));
161    hri = hris.next().getRegion();
162    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[5]));
163    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[6]));
164    hri = hris.next().getRegion();
165    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[6]));
166    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[7]));
167    hri = hris.next().getRegion();
168    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[7]));
169    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[8]));
170    hri = hris.next().getRegion();
171    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8]));
172    assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
173
174    // Now test using start/end with a number of regions
175
176    // Use 80 bit numbers to make sure we aren't limited
177    byte[] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
178    byte[] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
179
180    // Splitting into 10 regions, we expect (null,1) ... (9, null)
181    // with (1,2) (2,3) (3,4) (4,5) (5,6) (6,7) (7,8) (8,9) in the middle
182    expectedRegions = 10;
183    final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "_2");
184    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName2);
185    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
186    admin.createTable(builder.build(), startKey, endKey, expectedRegions).join();
187
188    regions = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName2).get();
189    assertEquals(
190      "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
191      expectedRegions, regions.size());
192    System.err.println("Found " + regions.size() + " regions");
193
194    hris = regions.iterator();
195    hri = hris.next().getRegion();
196    assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
197    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }));
198    hri = hris.next().getRegion();
199    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }));
200    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }));
201    hri = hris.next().getRegion();
202    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }));
203    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }));
204    hri = hris.next().getRegion();
205    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }));
206    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }));
207    hri = hris.next().getRegion();
208    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }));
209    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }));
210    hri = hris.next().getRegion();
211    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }));
212    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 }));
213    hri = hris.next().getRegion();
214    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 }));
215    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }));
216    hri = hris.next().getRegion();
217    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }));
218    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 }));
219    hri = hris.next().getRegion();
220    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 }));
221    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }));
222    hri = hris.next().getRegion();
223    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }));
224    assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
225
226    // Try once more with something that divides into something infinite
227    startKey = new byte[] { 0, 0, 0, 0, 0, 0 };
228    endKey = new byte[] { 1, 0, 0, 0, 0, 0 };
229
230    expectedRegions = 5;
231    final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "_3");
232    builder = TableDescriptorBuilder.newBuilder(tableName3);
233    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
234    admin.createTable(builder.build(), startKey, endKey, expectedRegions).join();
235
236    regions = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName3).get();
237    assertEquals(
238      "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
239      expectedRegions, regions.size());
240    System.err.println("Found " + regions.size() + " regions");
241
242    // Try an invalid case where there are duplicate split keys
243    splitKeys = new byte[][] { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 },
244      new byte[] { 3, 3, 3 }, new byte[] { 2, 2, 2 } };
245    final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "_4");
246    try {
247      createTableWithDefaultConf(tableName4, splitKeys);
248      fail("Should not be able to create this table because of " + "duplicate split keys");
249    } catch (CompletionException e) {
250      assertTrue(e.getCause() instanceof IllegalArgumentException);
251    }
252  }
253
254  @Test
255  public void testCreateTableWithOnlyEmptyStartRow() throws Exception {
256    byte[][] splitKeys = new byte[1][];
257    splitKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
258    try {
259      createTableWithDefaultConf(tableName, splitKeys);
260      fail("Test case should fail as empty split key is passed.");
261    } catch (CompletionException e) {
262      assertTrue(e.getCause() instanceof IllegalArgumentException);
263    }
264  }
265
266  @Test
267  public void testCreateTableWithEmptyRowInTheSplitKeys() throws Exception {
268    byte[][] splitKeys = new byte[3][];
269    splitKeys[0] = Bytes.toBytes("region1");
270    splitKeys[1] = HConstants.EMPTY_BYTE_ARRAY;
271    splitKeys[2] = Bytes.toBytes("region2");
272    try {
273      createTableWithDefaultConf(tableName, splitKeys);
274      fail("Test case should fail as empty split key is passed.");
275    } catch (CompletionException e) {
276      assertTrue(e.getCause() instanceof IllegalArgumentException);
277    }
278  }
279
280  @Test
281  public void testDeleteTable() throws Exception {
282    createTableWithDefaultConf(tableName);
283    assertTrue(admin.tableExists(tableName).get());
284    TEST_UTIL.getAdmin().disableTable(tableName);
285    admin.deleteTable(tableName).join();
286    assertFalse(admin.tableExists(tableName).get());
287  }
288
289  @Test
290  public void testTruncateTable() throws Exception {
291    testTruncateTable(tableName, false);
292  }
293
294  @Test
295  public void testTruncateTablePreservingSplits() throws Exception {
296    testTruncateTable(tableName, true);
297  }
298
299  private void testTruncateTable(final TableName tableName, boolean preserveSplits)
300    throws Exception {
301    byte[][] splitKeys = new byte[2][];
302    splitKeys[0] = Bytes.toBytes(4);
303    splitKeys[1] = Bytes.toBytes(8);
304
305    // Create & Fill the table
306    createTableWithDefaultConf(tableName, splitKeys);
307    AsyncTable<?> table = ASYNC_CONN.getTable(tableName);
308    int expectedRows = 10;
309    for (int i = 0; i < expectedRows; i++) {
310      byte[] data = Bytes.toBytes(String.valueOf(i));
311      Put put = new Put(data);
312      put.addColumn(FAMILY, null, data);
313      table.put(put).join();
314    }
315    assertEquals(10, table.scanAll(new Scan()).get().size());
316    assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
317
318    // Truncate & Verify
319    admin.disableTable(tableName).join();
320    admin.truncateTable(tableName, preserveSplits).join();
321    assertEquals(0, table.scanAll(new Scan()).get().size());
322    if (preserveSplits) {
323      assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
324    } else {
325      assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
326    }
327  }
328
329  @Test
330  public void testCloneTableSchema() throws Exception {
331    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
332    testCloneTableSchema(tableName, newTableName, false);
333  }
334
335  @Test
336  public void testCloneTableSchemaPreservingSplits() throws Exception {
337    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
338    testCloneTableSchema(tableName, newTableName, true);
339  }
340
341  private void testCloneTableSchema(final TableName tableName, final TableName newTableName,
342    boolean preserveSplits) throws Exception {
343    byte[][] splitKeys = new byte[2][];
344    splitKeys[0] = Bytes.toBytes(4);
345    splitKeys[1] = Bytes.toBytes(8);
346    int NUM_FAMILYS = 2;
347    int NUM_REGIONS = 3;
348    int BLOCK_SIZE = 1024;
349    int TTL = 86400;
350    boolean BLOCK_CACHE = false;
351
352    // Create the table
353    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
354      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0))
355      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_1).setBlocksize(BLOCK_SIZE)
356        .setBlockCacheEnabled(BLOCK_CACHE).setTimeToLive(TTL).build())
357      .build();
358    admin.createTable(tableDesc, splitKeys).join();
359
360    assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
361    assertTrue("Table should be created with splitKyes + 1 rows in META",
362      admin.isTableAvailable(tableName).get());
363
364    // Clone & Verify
365    admin.cloneTableSchema(tableName, newTableName, preserveSplits).join();
366    TableDescriptor newTableDesc = admin.getDescriptor(newTableName).get();
367
368    assertEquals(NUM_FAMILYS, newTableDesc.getColumnFamilyCount());
369    assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize());
370    assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled());
371    assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive());
372    // HBASE-26246 introduced persist of store file tracker into table descriptor
373    tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACKER_IMPL,
374      StoreFileTrackerFactory.getStoreFileTrackerName(TEST_UTIL.getConfiguration())).build();
375    TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc);
376
377    if (preserveSplits) {
378      assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size());
379      assertTrue("New table should be created with splitKyes + 1 rows in META",
380        admin.isTableAvailable(newTableName).get());
381    } else {
382      assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size());
383    }
384  }
385
386  @Test
387  public void testCloneTableSchemaWithNonExistentSourceTable() throws Exception {
388    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
389    // test for non-existent source table
390    try {
391      admin.cloneTableSchema(tableName, newTableName, false).join();
392      fail("Should have failed when source table doesn't exist.");
393    } catch (CompletionException e) {
394      assertTrue(e.getCause() instanceof TableNotFoundException);
395    }
396  }
397
398  @Test
399  public void testCloneTableSchemaWithExistentDestinationTable() throws Exception {
400    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
401    byte[] FAMILY_0 = Bytes.toBytes("cf0");
402    TEST_UTIL.createTable(tableName, FAMILY_0);
403    TEST_UTIL.createTable(newTableName, FAMILY_0);
404    // test for existent destination table
405    try {
406      admin.cloneTableSchema(tableName, newTableName, false).join();
407      fail("Should have failed when destination table exists.");
408    } catch (CompletionException e) {
409      assertTrue(e.getCause() instanceof TableExistsException);
410    }
411  }
412
413  @Test
414  public void testIsTableAvailableWithInexistantTable() throws Exception {
415    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
416    // test for inexistant table
417    assertFalse(admin.isTableAvailable(newTableName).get());
418  }
419}