001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
021import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL;
022import static org.junit.jupiter.api.Assertions.assertEquals;
023import static org.junit.jupiter.api.Assertions.assertFalse;
024import static org.junit.jupiter.api.Assertions.assertTrue;
025import static org.junit.jupiter.api.Assertions.fail;
026
027import java.util.Iterator;
028import java.util.List;
029import java.util.Optional;
030import java.util.concurrent.CompletionException;
031import java.util.function.Supplier;
032import org.apache.hadoop.hbase.ClientMetaTableAccessor;
033import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.HRegionLocation;
036import org.apache.hadoop.hbase.TableExistsException;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.TableNotFoundException;
039import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
040import org.apache.hadoop.hbase.testclassification.ClientTests;
041import org.apache.hadoop.hbase.testclassification.LargeTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.junit.jupiter.api.AfterAll;
044import org.junit.jupiter.api.BeforeAll;
045import org.junit.jupiter.api.Tag;
046import org.junit.jupiter.api.TestTemplate;
047
048/**
049 * Class to test asynchronous table admin operations.
050 * @see TestAsyncTableAdminApi2 This test and it used to be joined it was taking longer than our ten
051 *      minute timeout so they were split.
052 * @see TestAsyncTableAdminApi3 Another split out from this class so each runs under ten minutes.
053 */
054@Tag(LargeTests.TAG)
055@Tag(ClientTests.TAG)
056@HBaseParameterizedTestTemplate(name = "{index}: policy = {0}")
057public class TestAsyncTableAdminApi extends TestAsyncAdminBase {
058
059  public TestAsyncTableAdminApi(Supplier<AsyncAdmin> admin) {
060    super(admin);
061  }
062
063  @BeforeAll
064  public static void setUpBeforeClass() throws Exception {
065    TestAsyncAdminBase.setUpBeforeClass();
066  }
067
068  @AfterAll
069  public static void tearDownAfterClass() throws Exception {
070    TestAsyncAdminBase.tearDownAfterClass();
071  }
072
073  @TestTemplate
074  public void testCreateTable() throws Exception {
075    List<TableDescriptor> tables = admin.listTableDescriptors().get();
076    int numTables = tables.size();
077    createTableWithDefaultConf(tableName);
078    tables = admin.listTableDescriptors().get();
079    assertEquals(numTables + 1, tables.size());
080    assertTrue(TEST_UTIL.getHBaseCluster().getMaster().getTableStateManager()
081      .isTableState(tableName, TableState.State.ENABLED), "Table must be enabled.");
082    assertEquals(TableState.State.ENABLED, getStateFromMeta(tableName));
083  }
084
085  static TableState.State getStateFromMeta(TableName table) throws Exception {
086    Optional<TableState> state = ClientMetaTableAccessor
087      .getTableState(ASYNC_CONN.getTable(TableName.META_TABLE_NAME), table).get();
088    assertTrue(state.isPresent());
089    return state.get().getState();
090  }
091
092  @TestTemplate
093  public void testCreateTableNumberOfRegions() throws Exception {
094    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
095
096    createTableWithDefaultConf(tableName);
097    List<HRegionLocation> regionLocations =
098      ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
099    assertEquals(1, regionLocations.size(), "Table should have only 1 region");
100
101    final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "_2");
102    createTableWithDefaultConf(tableName2, new byte[][] { new byte[] { 42 } });
103    regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName2).get();
104    assertEquals(2, regionLocations.size(), "Table should have only 2 region");
105
106    final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "_3");
107    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName3);
108    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
109    admin.createTable(builder.build(), Bytes.toBytes("a"), Bytes.toBytes("z"), 3).join();
110    regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName3).get();
111    assertEquals(3, regionLocations.size(), "Table should have only 3 region");
112
113    final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "_4");
114    builder = TableDescriptorBuilder.newBuilder(tableName4);
115    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
116    try {
117      admin.createTable(builder.build(), Bytes.toBytes("a"), Bytes.toBytes("z"), 2).join();
118      fail("Should not be able to create a table with only 2 regions using this API.");
119    } catch (CompletionException e) {
120      assertTrue(e.getCause() instanceof IllegalArgumentException);
121    }
122
123    final TableName tableName5 = TableName.valueOf(tableName.getNameAsString() + "_5");
124    builder = TableDescriptorBuilder.newBuilder(tableName5);
125    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
126    admin.createTable(builder.build(), new byte[] { 1 }, new byte[] { 127 }, 16).join();
127    regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName5).get();
128    assertEquals(16, regionLocations.size(), "Table should have 16 region");
129  }
130
131  @TestTemplate
132  public void testCreateTableWithRegions() throws Exception {
133    byte[][] splitKeys = { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 }, new byte[] { 3, 3, 3 },
134      new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 }, new byte[] { 6, 6, 6 },
135      new byte[] { 7, 7, 7 }, new byte[] { 8, 8, 8 }, new byte[] { 9, 9, 9 }, };
136    int expectedRegions = splitKeys.length + 1;
137    createTableWithDefaultConf(tableName, splitKeys);
138
139    boolean tableAvailable = admin.isTableAvailable(tableName).get();
140    assertTrue(tableAvailable, "Table should be created with splitKyes + 1 rows in META");
141
142    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
143    List<HRegionLocation> regions =
144      ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
145    Iterator<HRegionLocation> hris = regions.iterator();
146
147    assertEquals(expectedRegions, regions.size(),
148      "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size());
149    System.err.println("Found " + regions.size() + " regions");
150
151    RegionInfo hri;
152    hris = regions.iterator();
153    hri = hris.next().getRegion();
154    assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
155    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[0]));
156    hri = hris.next().getRegion();
157    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[0]));
158    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[1]));
159    hri = hris.next().getRegion();
160    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[1]));
161    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[2]));
162    hri = hris.next().getRegion();
163    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[2]));
164    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[3]));
165    hri = hris.next().getRegion();
166    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[3]));
167    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[4]));
168    hri = hris.next().getRegion();
169    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[4]));
170    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[5]));
171    hri = hris.next().getRegion();
172    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[5]));
173    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[6]));
174    hri = hris.next().getRegion();
175    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[6]));
176    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[7]));
177    hri = hris.next().getRegion();
178    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[7]));
179    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[8]));
180    hri = hris.next().getRegion();
181    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8]));
182    assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
183
184    // Now test using start/end with a number of regions
185
186    // Use 80 bit numbers to make sure we aren't limited
187    byte[] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
188    byte[] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
189
190    // Splitting into 10 regions, we expect (null,1) ... (9, null)
191    // with (1,2) (2,3) (3,4) (4,5) (5,6) (6,7) (7,8) (8,9) in the middle
192    expectedRegions = 10;
193    final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "_2");
194    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName2);
195    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
196    admin.createTable(builder.build(), startKey, endKey, expectedRegions).join();
197
198    regions = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName2).get();
199    assertEquals(expectedRegions, regions.size(),
200      "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size());
201    System.err.println("Found " + regions.size() + " regions");
202
203    hris = regions.iterator();
204    hri = hris.next().getRegion();
205    assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
206    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }));
207    hri = hris.next().getRegion();
208    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }));
209    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }));
210    hri = hris.next().getRegion();
211    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }));
212    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }));
213    hri = hris.next().getRegion();
214    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }));
215    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }));
216    hri = hris.next().getRegion();
217    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }));
218    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }));
219    hri = hris.next().getRegion();
220    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }));
221    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 }));
222    hri = hris.next().getRegion();
223    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 }));
224    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }));
225    hri = hris.next().getRegion();
226    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }));
227    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 }));
228    hri = hris.next().getRegion();
229    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 }));
230    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }));
231    hri = hris.next().getRegion();
232    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }));
233    assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
234
235    // Try once more with something that divides into something infinite
236    startKey = new byte[] { 0, 0, 0, 0, 0, 0 };
237    endKey = new byte[] { 1, 0, 0, 0, 0, 0 };
238
239    expectedRegions = 5;
240    final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "_3");
241    builder = TableDescriptorBuilder.newBuilder(tableName3);
242    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
243    admin.createTable(builder.build(), startKey, endKey, expectedRegions).join();
244
245    regions = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName3).get();
246    assertEquals(expectedRegions, regions.size(),
247      "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size());
248    System.err.println("Found " + regions.size() + " regions");
249
250    // Try an invalid case where there are duplicate split keys
251    splitKeys = new byte[][] { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 },
252      new byte[] { 3, 3, 3 }, new byte[] { 2, 2, 2 } };
253    final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "_4");
254    try {
255      createTableWithDefaultConf(tableName4, splitKeys);
256      fail("Should not be able to create this table because of " + "duplicate split keys");
257    } catch (CompletionException e) {
258      assertTrue(e.getCause() instanceof IllegalArgumentException);
259    }
260  }
261
262  @TestTemplate
263  public void testCreateTableWithOnlyEmptyStartRow() throws Exception {
264    byte[][] splitKeys = new byte[1][];
265    splitKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
266    try {
267      createTableWithDefaultConf(tableName, splitKeys);
268      fail("Test case should fail as empty split key is passed.");
269    } catch (CompletionException e) {
270      assertTrue(e.getCause() instanceof IllegalArgumentException);
271    }
272  }
273
274  @TestTemplate
275  public void testCreateTableWithEmptyRowInTheSplitKeys() throws Exception {
276    byte[][] splitKeys = new byte[3][];
277    splitKeys[0] = Bytes.toBytes("region1");
278    splitKeys[1] = HConstants.EMPTY_BYTE_ARRAY;
279    splitKeys[2] = Bytes.toBytes("region2");
280    try {
281      createTableWithDefaultConf(tableName, splitKeys);
282      fail("Test case should fail as empty split key is passed.");
283    } catch (CompletionException e) {
284      assertTrue(e.getCause() instanceof IllegalArgumentException);
285    }
286  }
287
288  @TestTemplate
289  public void testDeleteTable() throws Exception {
290    createTableWithDefaultConf(tableName);
291    assertTrue(admin.tableExists(tableName).get());
292    TEST_UTIL.getAdmin().disableTable(tableName);
293    admin.deleteTable(tableName).join();
294    assertFalse(admin.tableExists(tableName).get());
295  }
296
297  @TestTemplate
298  public void testTruncateTable() throws Exception {
299    testTruncateTable(tableName, false);
300  }
301
302  @TestTemplate
303  public void testTruncateTablePreservingSplits() throws Exception {
304    testTruncateTable(tableName, true);
305  }
306
307  private void testTruncateTable(final TableName tableName, boolean preserveSplits)
308    throws Exception {
309    byte[][] splitKeys = new byte[2][];
310    splitKeys[0] = Bytes.toBytes(4);
311    splitKeys[1] = Bytes.toBytes(8);
312
313    // Create & Fill the table
314    createTableWithDefaultConf(tableName, splitKeys);
315    AsyncTable<?> table = ASYNC_CONN.getTable(tableName);
316    int expectedRows = 10;
317    for (int i = 0; i < expectedRows; i++) {
318      byte[] data = Bytes.toBytes(String.valueOf(i));
319      Put put = new Put(data);
320      put.addColumn(FAMILY, null, data);
321      table.put(put).join();
322    }
323    assertEquals(10, table.scanAll(new Scan()).get().size());
324    assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
325
326    // Truncate & Verify
327    admin.disableTable(tableName).join();
328    admin.truncateTable(tableName, preserveSplits).join();
329    assertEquals(0, table.scanAll(new Scan()).get().size());
330    if (preserveSplits) {
331      assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
332    } else {
333      assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
334    }
335  }
336
337  @TestTemplate
338  public void testCloneTableSchema() throws Exception {
339    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
340    testCloneTableSchema(tableName, newTableName, false);
341  }
342
343  @TestTemplate
344  public void testCloneTableSchemaPreservingSplits() throws Exception {
345    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
346    testCloneTableSchema(tableName, newTableName, true);
347  }
348
349  private void testCloneTableSchema(final TableName tableName, final TableName newTableName,
350    boolean preserveSplits) throws Exception {
351    byte[][] splitKeys = new byte[2][];
352    splitKeys[0] = Bytes.toBytes(4);
353    splitKeys[1] = Bytes.toBytes(8);
354    int NUM_FAMILYS = 2;
355    int NUM_REGIONS = 3;
356    int BLOCK_SIZE = 1024;
357    int TTL = 86400;
358    boolean BLOCK_CACHE = false;
359
360    // Create the table
361    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
362      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0))
363      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_1).setBlocksize(BLOCK_SIZE)
364        .setBlockCacheEnabled(BLOCK_CACHE).setTimeToLive(TTL).build())
365      .build();
366    admin.createTable(tableDesc, splitKeys).join();
367
368    assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
369    assertTrue(admin.isTableAvailable(tableName).get(),
370      "Table should be created with splitKyes + 1 rows in META");
371
372    // Clone & Verify
373    admin.cloneTableSchema(tableName, newTableName, preserveSplits).join();
374    TableDescriptor newTableDesc = admin.getDescriptor(newTableName).get();
375
376    assertEquals(NUM_FAMILYS, newTableDesc.getColumnFamilyCount());
377    assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize());
378    assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled());
379    assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive());
380    // HBASE-26246 introduced persist of store file tracker into table descriptor
381    tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACKER_IMPL,
382      StoreFileTrackerFactory.getStoreFileTrackerName(TEST_UTIL.getConfiguration())).build();
383    TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc);
384
385    if (preserveSplits) {
386      assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size());
387      assertTrue(admin.isTableAvailable(newTableName).get(),
388        "New table should be created with splitKyes + 1 rows in META");
389    } else {
390      assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size());
391    }
392  }
393
394  @TestTemplate
395  public void testCloneTableSchemaWithNonExistentSourceTable() throws Exception {
396    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
397    // test for non-existent source table
398    try {
399      admin.cloneTableSchema(tableName, newTableName, false).join();
400      fail("Should have failed when source table doesn't exist.");
401    } catch (CompletionException e) {
402      assertTrue(e.getCause() instanceof TableNotFoundException);
403    }
404  }
405
406  @TestTemplate
407  public void testCloneTableSchemaWithExistentDestinationTable() throws Exception {
408    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
409    byte[] FAMILY_0 = Bytes.toBytes("cf0");
410    TEST_UTIL.createTable(tableName, FAMILY_0);
411    TEST_UTIL.createTable(newTableName, FAMILY_0);
412    // test for existent destination table
413    try {
414      admin.cloneTableSchema(tableName, newTableName, false).join();
415      fail("Should have failed when destination table exists.");
416    } catch (CompletionException e) {
417      assertTrue(e.getCause() instanceof TableExistsException);
418    }
419  }
420
421  @TestTemplate
422  public void testIsTableAvailableWithInexistantTable() throws Exception {
423    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
424    // test for inexistant table
425    assertFalse(admin.isTableAvailable(newTableName).get());
426  }
427}