001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.client; 019 020import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; 021import static org.junit.Assert.assertEquals; 022import static org.junit.Assert.assertFalse; 023import static org.junit.Assert.assertTrue; 024import static org.junit.Assert.fail; 025 026import java.io.IOException; 027import java.util.ArrayList; 028import java.util.HashMap; 029import java.util.Iterator; 030import java.util.List; 031import java.util.Map; 032import java.util.Optional; 033import java.util.concurrent.CompletionException; 034import org.apache.hadoop.hbase.AsyncMetaTableAccessor; 035import org.apache.hadoop.hbase.HBaseClassTestRule; 036import org.apache.hadoop.hbase.HConstants; 037import org.apache.hadoop.hbase.HRegionLocation; 038import org.apache.hadoop.hbase.ServerName; 039import org.apache.hadoop.hbase.TableName; 040import org.apache.hadoop.hbase.master.LoadBalancer; 041import org.apache.hadoop.hbase.testclassification.ClientTests; 042import org.apache.hadoop.hbase.testclassification.LargeTests; 043import org.apache.hadoop.hbase.util.Bytes; 044import org.junit.ClassRule; 045import org.junit.Test; 046import org.junit.experimental.categories.Category; 047import org.junit.runner.RunWith; 048import org.junit.runners.Parameterized; 049 050/** 051 * Class to test asynchronous table admin operations. 052 * @see TestAsyncTableAdminApi2 This test and it used to be joined it was taking longer than our 053 * ten minute timeout so they were split. 054 * @see TestAsyncTableAdminApi3 Another split out from this class so each runs under ten minutes. 055 */ 056@RunWith(Parameterized.class) 057@Category({ LargeTests.class, ClientTests.class }) 058public class TestAsyncTableAdminApi extends TestAsyncAdminBase { 059 060 @ClassRule 061 public static final HBaseClassTestRule CLASS_RULE = 062 HBaseClassTestRule.forClass(TestAsyncTableAdminApi.class); 063 064 @Test 065 public void testCreateTable() throws Exception { 066 List<TableDescriptor> tables = admin.listTableDescriptors().get(); 067 int numTables = tables.size(); 068 createTableWithDefaultConf(tableName); 069 tables = admin.listTableDescriptors().get(); 070 assertEquals(numTables + 1, tables.size()); 071 assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster().getMaster() 072 .getTableStateManager().isTableState(tableName, TableState.State.ENABLED)); 073 assertEquals(TableState.State.ENABLED, getStateFromMeta(tableName)); 074 } 075 076 static TableState.State getStateFromMeta(TableName table) throws Exception { 077 Optional<TableState> state = AsyncMetaTableAccessor 078 .getTableState(ASYNC_CONN.getTable(TableName.META_TABLE_NAME), table).get(); 079 assertTrue(state.isPresent()); 080 return state.get().getState(); 081 } 082 083 @Test 084 public void testCreateTableNumberOfRegions() throws Exception { 085 AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME); 086 087 createTableWithDefaultConf(tableName); 088 List<HRegionLocation> regionLocations = 089 AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName)).get(); 090 assertEquals("Table should have only 1 region", 1, regionLocations.size()); 091 092 final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "_2"); 093 createTableWithDefaultConf(tableName2, new byte[][] { new byte[] { 42 } }); 094 regionLocations = 095 AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName2)).get(); 096 assertEquals("Table should have only 2 region", 2, regionLocations.size()); 097 098 final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "_3"); 099 TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName3); 100 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)); 101 admin.createTable(builder.build(), "a".getBytes(), "z".getBytes(), 3).join(); 102 regionLocations = 103 AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName3)).get(); 104 assertEquals("Table should have only 3 region", 3, regionLocations.size()); 105 106 final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "_4"); 107 builder = TableDescriptorBuilder.newBuilder(tableName4); 108 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)); 109 try { 110 admin.createTable(builder.build(), "a".getBytes(), "z".getBytes(), 2).join(); 111 fail("Should not be able to create a table with only 2 regions using this API."); 112 } catch (CompletionException e) { 113 assertTrue(e.getCause() instanceof IllegalArgumentException); 114 } 115 116 final TableName tableName5 = TableName.valueOf(tableName.getNameAsString() + "_5"); 117 builder = TableDescriptorBuilder.newBuilder(tableName5); 118 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)); 119 admin.createTable(builder.build(), new byte[] { 1 }, new byte[] { 127 }, 16).join(); 120 regionLocations = 121 AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName5)).get(); 122 assertEquals("Table should have 16 region", 16, regionLocations.size()); 123 } 124 125 @Test 126 public void testCreateTableWithRegions() throws Exception { 127 byte[][] splitKeys = { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 }, new byte[] { 3, 3, 3 }, 128 new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 }, new byte[] { 6, 6, 6 }, 129 new byte[] { 7, 7, 7 }, new byte[] { 8, 8, 8 }, new byte[] { 9, 9, 9 }, }; 130 int expectedRegions = splitKeys.length + 1; 131 boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration()); 132 createTableWithDefaultConf(tableName, splitKeys); 133 134 boolean tableAvailable = admin.isTableAvailable(tableName, splitKeys).get(); 135 assertTrue("Table should be created with splitKyes + 1 rows in META", tableAvailable); 136 137 AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME); 138 List<HRegionLocation> regions = 139 AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName)).get(); 140 Iterator<HRegionLocation> hris = regions.iterator(); 141 142 assertEquals( 143 "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), 144 expectedRegions, regions.size()); 145 System.err.println("Found " + regions.size() + " regions"); 146 147 RegionInfo hri; 148 hris = regions.iterator(); 149 hri = hris.next().getRegion(); 150 assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0); 151 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[0])); 152 hri = hris.next().getRegion(); 153 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[0])); 154 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[1])); 155 hri = hris.next().getRegion(); 156 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[1])); 157 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[2])); 158 hri = hris.next().getRegion(); 159 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[2])); 160 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[3])); 161 hri = hris.next().getRegion(); 162 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[3])); 163 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[4])); 164 hri = hris.next().getRegion(); 165 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[4])); 166 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[5])); 167 hri = hris.next().getRegion(); 168 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[5])); 169 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[6])); 170 hri = hris.next().getRegion(); 171 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[6])); 172 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[7])); 173 hri = hris.next().getRegion(); 174 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[7])); 175 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[8])); 176 hri = hris.next().getRegion(); 177 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8])); 178 assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0); 179 if (tablesOnMaster) { 180 verifyRoundRobinDistribution(regions, expectedRegions); 181 } 182 183 // Now test using start/end with a number of regions 184 185 // Use 80 bit numbers to make sure we aren't limited 186 byte[] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; 187 byte[] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; 188 189 // Splitting into 10 regions, we expect (null,1) ... (9, null) 190 // with (1,2) (2,3) (3,4) (4,5) (5,6) (6,7) (7,8) (8,9) in the middle 191 expectedRegions = 10; 192 final TableName tableName2 = TableName.valueOf(tableName.getNameAsString() + "_2"); 193 TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName2); 194 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)); 195 admin.createTable(builder.build(), startKey, endKey, expectedRegions).join(); 196 197 regions = 198 AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName2)).get(); 199 assertEquals( 200 "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), 201 expectedRegions, regions.size()); 202 System.err.println("Found " + regions.size() + " regions"); 203 204 hris = regions.iterator(); 205 hri = hris.next().getRegion(); 206 assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0); 207 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 })); 208 hri = hris.next().getRegion(); 209 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 })); 210 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 })); 211 hri = hris.next().getRegion(); 212 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 })); 213 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 })); 214 hri = hris.next().getRegion(); 215 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 })); 216 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 })); 217 hri = hris.next().getRegion(); 218 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 })); 219 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 })); 220 hri = hris.next().getRegion(); 221 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 })); 222 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 })); 223 hri = hris.next().getRegion(); 224 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6 })); 225 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 })); 226 hri = hris.next().getRegion(); 227 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 })); 228 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 })); 229 hri = hris.next().getRegion(); 230 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8 })); 231 assertTrue(Bytes.equals(hri.getEndKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 })); 232 hri = hris.next().getRegion(); 233 assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 })); 234 assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0); 235 if (tablesOnMaster) { 236 // This don't work if master is not carrying regions. FIX. TODO. 237 verifyRoundRobinDistribution(regions, expectedRegions); 238 } 239 240 // Try once more with something that divides into something infinite 241 startKey = new byte[] { 0, 0, 0, 0, 0, 0 }; 242 endKey = new byte[] { 1, 0, 0, 0, 0, 0 }; 243 244 expectedRegions = 5; 245 final TableName tableName3 = TableName.valueOf(tableName.getNameAsString() + "_3"); 246 builder = TableDescriptorBuilder.newBuilder(tableName3); 247 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)); 248 admin.createTable(builder.build(), startKey, endKey, expectedRegions).join(); 249 250 regions = 251 AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, Optional.of(tableName3)).get(); 252 assertEquals( 253 "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), 254 expectedRegions, regions.size()); 255 System.err.println("Found " + regions.size() + " regions"); 256 if (tablesOnMaster) { 257 // This don't work if master is not carrying regions. FIX. TODO. 258 verifyRoundRobinDistribution(regions, expectedRegions); 259 } 260 261 // Try an invalid case where there are duplicate split keys 262 splitKeys = new byte[][] { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 }, 263 new byte[] { 3, 3, 3 }, new byte[] { 2, 2, 2 } }; 264 final TableName tableName4 = TableName.valueOf(tableName.getNameAsString() + "_4"); 265 try { 266 createTableWithDefaultConf(tableName4, splitKeys); 267 fail("Should not be able to create this table because of " + "duplicate split keys"); 268 } catch (CompletionException e) { 269 assertTrue(e.getCause() instanceof IllegalArgumentException); 270 } 271 } 272 273 private void verifyRoundRobinDistribution(List<HRegionLocation> regions, int expectedRegions) 274 throws IOException { 275 int numRS = ((ClusterConnection) TEST_UTIL.getConnection()).getCurrentNrHRS(); 276 277 Map<ServerName, List<RegionInfo>> server2Regions = new HashMap<>(); 278 regions.stream().forEach((loc) -> { 279 ServerName server = loc.getServerName(); 280 server2Regions.computeIfAbsent(server, (s) -> new ArrayList<>()).add(loc.getRegion()); 281 }); 282 if (numRS >= 2) { 283 // Ignore the master region server, 284 // which contains less regions by intention. 285 numRS--; 286 } 287 float average = (float) expectedRegions / numRS; 288 int min = (int) Math.floor(average); 289 int max = (int) Math.ceil(average); 290 server2Regions.values().forEach((regionList) -> { 291 assertTrue(regionList.size() == min || regionList.size() == max); 292 }); 293 } 294 295 @Test 296 public void testCreateTableWithOnlyEmptyStartRow() throws Exception { 297 byte[][] splitKeys = new byte[1][]; 298 splitKeys[0] = HConstants.EMPTY_BYTE_ARRAY; 299 try { 300 createTableWithDefaultConf(tableName, splitKeys); 301 fail("Test case should fail as empty split key is passed."); 302 } catch (CompletionException e) { 303 assertTrue(e.getCause() instanceof IllegalArgumentException); 304 } 305 } 306 307 @Test 308 public void testCreateTableWithEmptyRowInTheSplitKeys() throws Exception { 309 byte[][] splitKeys = new byte[3][]; 310 splitKeys[0] = "region1".getBytes(); 311 splitKeys[1] = HConstants.EMPTY_BYTE_ARRAY; 312 splitKeys[2] = "region2".getBytes(); 313 try { 314 createTableWithDefaultConf(tableName, splitKeys); 315 fail("Test case should fail as empty split key is passed."); 316 } catch (CompletionException e) { 317 assertTrue(e.getCause() instanceof IllegalArgumentException); 318 } 319 } 320 321 @Test 322 public void testDeleteTable() throws Exception { 323 createTableWithDefaultConf(tableName); 324 assertTrue(admin.tableExists(tableName).get()); 325 TEST_UTIL.getAdmin().disableTable(tableName); 326 admin.deleteTable(tableName).join(); 327 assertFalse(admin.tableExists(tableName).get()); 328 } 329 330 @Test 331 public void testTruncateTable() throws Exception { 332 testTruncateTable(tableName, false); 333 } 334 335 @Test 336 public void testTruncateTablePreservingSplits() throws Exception { 337 testTruncateTable(tableName, true); 338 } 339 340 private void testTruncateTable(final TableName tableName, boolean preserveSplits) 341 throws Exception { 342 byte[][] splitKeys = new byte[2][]; 343 splitKeys[0] = Bytes.toBytes(4); 344 splitKeys[1] = Bytes.toBytes(8); 345 346 // Create & Fill the table 347 createTableWithDefaultConf(tableName, splitKeys); 348 AsyncTable<?> table = ASYNC_CONN.getTable(tableName); 349 int expectedRows = 10; 350 for (int i = 0; i < expectedRows; i++) { 351 byte[] data = Bytes.toBytes(String.valueOf(i)); 352 Put put = new Put(data); 353 put.addColumn(FAMILY, null, data); 354 table.put(put).join(); 355 } 356 assertEquals(10, table.scanAll(new Scan()).get().size()); 357 assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size()); 358 359 // Truncate & Verify 360 admin.disableTable(tableName).join(); 361 admin.truncateTable(tableName, preserveSplits).join(); 362 assertEquals(0, table.scanAll(new Scan()).get().size()); 363 if (preserveSplits) { 364 assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size()); 365 } else { 366 assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(tableName).size()); 367 } 368 } 369}