001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
021import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL;
022import static org.junit.jupiter.api.Assertions.assertEquals;
023import static org.junit.jupiter.api.Assertions.assertFalse;
024import static org.junit.jupiter.api.Assertions.assertTrue;
025import static org.junit.jupiter.api.Assertions.fail;
026
027import java.util.Iterator;
028import java.util.List;
029import java.util.Optional;
030import java.util.concurrent.CompletionException;
031import java.util.function.Supplier;
032import org.apache.hadoop.hbase.ClientMetaTableAccessor;
033import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.HRegionLocation;
036import org.apache.hadoop.hbase.TableExistsException;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.TableNotFoundException;
039import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
040import org.apache.hadoop.hbase.testclassification.ClientTests;
041import org.apache.hadoop.hbase.testclassification.LargeTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.junit.jupiter.api.AfterAll;
044import org.junit.jupiter.api.BeforeAll;
045import org.junit.jupiter.api.Tag;
046import org.junit.jupiter.api.TestTemplate;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050/**
051 * Class to test asynchronous table admin operations.
052 * @see TestAsyncTableAdminApi2 This test and it used to be joined it was taking longer than our ten
053 *      minute timeout so they were split.
054 * @see TestAsyncTableAdminApi3 Another split out from this class so each runs under ten minutes.
055 */
056@Tag(LargeTests.TAG)
057@Tag(ClientTests.TAG)
058@HBaseParameterizedTestTemplate(name = "{index}: policy = {0}")
059public class TestAsyncTableAdminApi extends TestAsyncAdminBase {
060
061  private static final Logger LOG = LoggerFactory.getLogger(TestAsyncTableAdminApi.class);
062
063  public TestAsyncTableAdminApi(Supplier<AsyncAdmin> admin) {
064    super(admin);
065  }
066
067  @BeforeAll
068  public static void setUpBeforeClass() throws Exception {
069    TestAsyncAdminBase.setUpBeforeClass();
070  }
071
072  @AfterAll
073  public static void tearDownAfterClass() throws Exception {
074    TestAsyncAdminBase.tearDownAfterClass();
075  }
076
077  @TestTemplate
078  public void testCreateTable() throws Exception {
079    List<TableDescriptor> tables = admin.listTableDescriptors().get();
080    int numTables = tables.size();
081    createTableWithDefaultConf(tableName);
082    tables = admin.listTableDescriptors().get();
083    assertEquals(numTables + 1, tables.size());
084    assertTrue(TEST_UTIL.getHBaseCluster().getMaster().getTableStateManager()
085      .isTableState(tableName, TableState.State.ENABLED), "Table must be enabled.");
086    assertEquals(TableState.State.ENABLED, getStateFromMeta(tableName));
087  }
088
089  static TableState.State getStateFromMeta(TableName table) throws Exception {
090    Optional<TableState> state = ClientMetaTableAccessor
091      .getTableState(ASYNC_CONN.getTable(TableName.META_TABLE_NAME), table).get();
092    assertTrue(state.isPresent());
093    return state.get().getState();
094  }
095
096  @TestTemplate
097  public void testCreateTableNumberOfRegions() throws Exception {
098    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
099
100    createTableWithDefaultConf(tableName);
101    List<HRegionLocation> regionLocations =
102      ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
103    assertEquals(1, regionLocations.size(), "Table should have only 1 region");
104
105    final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "_2");
106    createTableWithDefaultConf(tableName2, new byte[][] { new byte[] { 42 } });
107    regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName2).get();
108    assertEquals(2, regionLocations.size(), "Table should have only 2 region");
109
110    final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "_3");
111    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName3);
112    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
113    admin.createTable(builder.build(), Bytes.toBytes("a"), Bytes.toBytes("z"), 3).join();
114    regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName3).get();
115    assertEquals(3, regionLocations.size(), "Table should have only 3 region");
116
117    final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "_4");
118    builder = TableDescriptorBuilder.newBuilder(tableName4);
119    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
120    try {
121      admin.createTable(builder.build(), Bytes.toBytes("a"), Bytes.toBytes("z"), 2).join();
122      fail("Should not be able to create a table with only 2 regions using this API.");
123    } catch (CompletionException e) {
124      assertTrue(e.getCause() instanceof IllegalArgumentException);
125    }
126
127    final TableName tableName5 = TableName.valueOf(tableName.getNameAsString() + "_5");
128    builder = TableDescriptorBuilder.newBuilder(tableName5);
129    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
130    admin.createTable(builder.build(), new byte[] { 1 }, new byte[] { 127 }, 16).join();
131    regionLocations = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName5).get();
132    assertEquals(16, regionLocations.size(), "Table should have 16 region");
133  }
134
135  @TestTemplate
136  public void testCreateTableWithRegions() throws Exception {
137    byte[][] splitKeys = { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 }, new byte[] { 3, 3, 3 },
138      new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 }, new byte[] { 6, 6, 6 },
139      new byte[] { 7, 7, 7 }, new byte[] { 8, 8, 8 }, new byte[] { 9, 9, 9 }, };
140    int expectedRegions = splitKeys.length + 1;
141    createTableWithDefaultConf(tableName, splitKeys);
142
143    boolean tableAvailable = admin.isTableAvailable(tableName).get();
144    assertTrue(tableAvailable, "Table should be created with splitKyes + 1 rows in META");
145
146    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
147    List<HRegionLocation> regions =
148      ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
149    Iterator<HRegionLocation> hris = regions.iterator();
150
151    assertEquals(expectedRegions, regions.size(),
152      "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size());
153    LOG.info("Found " + regions.size() + " regions");
154
155    RegionInfo hri;
156    hris = regions.iterator();
157    hri = hris.next().getRegion();
158    assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
159    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[0]));
160    hri = hris.next().getRegion();
161    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[0]));
162    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[1]));
163    hri = hris.next().getRegion();
164    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[1]));
165    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[2]));
166    hri = hris.next().getRegion();
167    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[2]));
168    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[3]));
169    hri = hris.next().getRegion();
170    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[3]));
171    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[4]));
172    hri = hris.next().getRegion();
173    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[4]));
174    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[5]));
175    hri = hris.next().getRegion();
176    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[5]));
177    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[6]));
178    hri = hris.next().getRegion();
179    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[6]));
180    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[7]));
181    hri = hris.next().getRegion();
182    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[7]));
183    assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[8]));
184    hri = hris.next().getRegion();
185    assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8]));
186    assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
187
188    // Now test using start/end with a number of regions
189
190    // Use 80 bit numbers to make sure we aren't limited
191    byte[] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
192    byte[] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
193
194    // Splitting into 10 regions, we expect (null,1) ... (9, null)
195    // with (1,2) (2,3) (3,4) (4,5) (5,6) (6,7) (7,8) (8,9) in the middle
196    expectedRegions = 10;
197    final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "_2");
198    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName2);
199    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
200    admin.createTable(builder.build(), startKey, endKey, expectedRegions).join();
201
202    regions = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName2).get();
203    assertEquals(expectedRegions, regions.size(),
204      "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size());
205    LOG.info("Found " + regions.size() + " regions");
206
207    hris = regions.iterator();
208    hri = hris.next().getRegion();
209    assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
210    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }));
211    hri = hris.next().getRegion();
212    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }));
213    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }));
214    hri = hris.next().getRegion();
215    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 }));
216    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }));
217    hri = hris.next().getRegion();
218    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }));
219    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }));
220    hri = hris.next().getRegion();
221    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }));
222    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }));
223    hri = hris.next().getRegion();
224    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }));
225    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 }));
226    hri = hris.next().getRegion();
227    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 }));
228    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }));
229    hri = hris.next().getRegion();
230    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }));
231    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 }));
232    hri = hris.next().getRegion();
233    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 }));
234    assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }));
235    hri = hris.next().getRegion();
236    assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }));
237    assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
238
239    // Try once more with something that divides into something infinite
240    startKey = new byte[] { 0, 0, 0, 0, 0, 0 };
241    endKey = new byte[] { 1, 0, 0, 0, 0, 0 };
242
243    expectedRegions = 5;
244    final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "_3");
245    builder = TableDescriptorBuilder.newBuilder(tableName3);
246    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
247    admin.createTable(builder.build(), startKey, endKey, expectedRegions).join();
248
249    regions = ClientMetaTableAccessor.getTableHRegionLocations(metaTable, tableName3).get();
250    assertEquals(expectedRegions, regions.size(),
251      "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size());
252    LOG.info("Found " + regions.size() + " regions");
253
254    // Try an invalid case where there are duplicate split keys
255    splitKeys = new byte[][] { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 },
256      new byte[] { 3, 3, 3 }, new byte[] { 2, 2, 2 } };
257    final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "_4");
258    try {
259      createTableWithDefaultConf(tableName4, splitKeys);
260      fail("Should not be able to create this table because of " + "duplicate split keys");
261    } catch (CompletionException e) {
262      assertTrue(e.getCause() instanceof IllegalArgumentException);
263    }
264  }
265
266  @TestTemplate
267  public void testCreateTableWithOnlyEmptyStartRow() throws Exception {
268    byte[][] splitKeys = new byte[1][];
269    splitKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
270    try {
271      createTableWithDefaultConf(tableName, splitKeys);
272      fail("Test case should fail as empty split key is passed.");
273    } catch (CompletionException e) {
274      assertTrue(e.getCause() instanceof IllegalArgumentException);
275    }
276  }
277
278  @TestTemplate
279  public void testCreateTableWithEmptyRowInTheSplitKeys() throws Exception {
280    byte[][] splitKeys = new byte[3][];
281    splitKeys[0] = Bytes.toBytes("region1");
282    splitKeys[1] = HConstants.EMPTY_BYTE_ARRAY;
283    splitKeys[2] = Bytes.toBytes("region2");
284    try {
285      createTableWithDefaultConf(tableName, splitKeys);
286      fail("Test case should fail as empty split key is passed.");
287    } catch (CompletionException e) {
288      assertTrue(e.getCause() instanceof IllegalArgumentException);
289    }
290  }
291
292  @TestTemplate
293  public void testDeleteTable() throws Exception {
294    createTableWithDefaultConf(tableName);
295    assertTrue(admin.tableExists(tableName).get());
296    TEST_UTIL.getAdmin().disableTable(tableName);
297    admin.deleteTable(tableName).join();
298    assertFalse(admin.tableExists(tableName).get());
299  }
300
301  @TestTemplate
302  public void testTruncateTable() throws Exception {
303    testTruncateTable(tableName, false);
304  }
305
306  @TestTemplate
307  public void testTruncateTablePreservingSplits() throws Exception {
308    testTruncateTable(tableName, true);
309  }
310
311  private void testTruncateTable(final TableName tableName, boolean preserveSplits)
312    throws Exception {
313    byte[][] splitKeys = new byte[2][];
314    splitKeys[0] = Bytes.toBytes(4);
315    splitKeys[1] = Bytes.toBytes(8);
316
317    // Create & Fill the table
318    createTableWithDefaultConf(tableName, splitKeys);
319    AsyncTable<?> table = ASYNC_CONN.getTable(tableName);
320    int expectedRows = 10;
321    for (int i = 0; i < expectedRows; i++) {
322      byte[] data = Bytes.toBytes(String.valueOf(i));
323      Put put = new Put(data);
324      put.addColumn(FAMILY, null, data);
325      table.put(put).join();
326    }
327    assertEquals(10, table.scanAll(new Scan()).get().size());
328    assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
329
330    // Truncate & Verify
331    admin.disableTable(tableName).join();
332    admin.truncateTable(tableName, preserveSplits).join();
333    assertEquals(0, table.scanAll(new Scan()).get().size());
334    if (preserveSplits) {
335      assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
336    } else {
337      assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
338    }
339  }
340
341  @TestTemplate
342  public void testCloneTableSchema() throws Exception {
343    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
344    testCloneTableSchema(tableName, newTableName, false);
345  }
346
347  @TestTemplate
348  public void testCloneTableSchemaPreservingSplits() throws Exception {
349    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
350    testCloneTableSchema(tableName, newTableName, true);
351  }
352
353  private void testCloneTableSchema(final TableName tableName, final TableName newTableName,
354    boolean preserveSplits) throws Exception {
355    byte[][] splitKeys = new byte[2][];
356    splitKeys[0] = Bytes.toBytes(4);
357    splitKeys[1] = Bytes.toBytes(8);
358    int NUM_FAMILYS = 2;
359    int NUM_REGIONS = 3;
360    int BLOCK_SIZE = 1024;
361    int TTL = 86400;
362    boolean BLOCK_CACHE = false;
363
364    // Create the table
365    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
366      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0))
367      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_1).setBlocksize(BLOCK_SIZE)
368        .setBlockCacheEnabled(BLOCK_CACHE).setTimeToLive(TTL).build())
369      .build();
370    admin.createTable(tableDesc, splitKeys).join();
371
372    assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
373    assertTrue(admin.isTableAvailable(tableName).get(),
374      "Table should be created with splitKyes + 1 rows in META");
375
376    // Clone & Verify
377    admin.cloneTableSchema(tableName, newTableName, preserveSplits).join();
378    TableDescriptor newTableDesc = admin.getDescriptor(newTableName).get();
379
380    assertEquals(NUM_FAMILYS, newTableDesc.getColumnFamilyCount());
381    assertEquals(BLOCK_SIZE, newTableDesc.getColumnFamily(FAMILY_1).getBlocksize());
382    assertEquals(BLOCK_CACHE, newTableDesc.getColumnFamily(FAMILY_1).isBlockCacheEnabled());
383    assertEquals(TTL, newTableDesc.getColumnFamily(FAMILY_1).getTimeToLive());
384    // HBASE-26246 introduced persist of store file tracker into table descriptor
385    tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).setValue(TRACKER_IMPL,
386      StoreFileTrackerFactory.getStoreFileTrackerName(TEST_UTIL.getConfiguration())).build();
387    TEST_UTIL.verifyTableDescriptorIgnoreTableName(tableDesc, newTableDesc);
388
389    if (preserveSplits) {
390      assertEquals(NUM_REGIONS, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size());
391      assertTrue(admin.isTableAvailable(newTableName).get(),
392        "New table should be created with splitKyes + 1 rows in META");
393    } else {
394      assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(newTableName).size());
395    }
396  }
397
398  @TestTemplate
399  public void testCloneTableSchemaWithNonExistentSourceTable() throws Exception {
400    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
401    // test for non-existent source table
402    try {
403      admin.cloneTableSchema(tableName, newTableName, false).join();
404      fail("Should have failed when source table doesn't exist.");
405    } catch (CompletionException e) {
406      assertTrue(e.getCause() instanceof TableNotFoundException);
407    }
408  }
409
410  @TestTemplate
411  public void testCloneTableSchemaWithExistentDestinationTable() throws Exception {
412    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
413    byte[] FAMILY_0 = Bytes.toBytes("cf0");
414    TEST_UTIL.createTable(tableName, FAMILY_0);
415    TEST_UTIL.createTable(newTableName, FAMILY_0);
416    // test for existent destination table
417    try {
418      admin.cloneTableSchema(tableName, newTableName, false).join();
419      fail("Should have failed when destination table exists.");
420    } catch (CompletionException e) {
421      assertTrue(e.getCause() instanceof TableExistsException);
422    }
423  }
424
425  @TestTemplate
426  public void testIsTableAvailableWithInexistantTable() throws Exception {
427    final TableName newTableName = TableName.valueOf(tableName.getNameAsString() + "_new");
428    // test for inexistant table
429    assertFalse(admin.isTableAvailable(newTableName).get());
430  }
431}