001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.thrift; 019 020import static org.apache.hadoop.hbase.thrift.Constants.COALESCE_INC_KEY; 021import static org.junit.Assert.assertArrayEquals; 022import static org.junit.Assert.assertEquals; 023import static org.junit.Assert.assertFalse; 024import static org.junit.Assert.assertTrue; 025import static org.junit.Assert.fail; 026 027import java.io.IOException; 028import java.nio.ByteBuffer; 029import java.util.ArrayList; 030import java.util.Collection; 031import java.util.HashMap; 032import java.util.List; 033import java.util.Map; 034import org.apache.hadoop.conf.Configuration; 035import org.apache.hadoop.hbase.CompatibilityFactory; 036import org.apache.hadoop.hbase.HBaseClassTestRule; 037import org.apache.hadoop.hbase.HBaseTestingUtility; 038import org.apache.hadoop.hbase.HColumnDescriptor; 039import org.apache.hadoop.hbase.HConstants; 040import org.apache.hadoop.hbase.HRegionInfo; 041import org.apache.hadoop.hbase.HTableDescriptor; 042import org.apache.hadoop.hbase.TableName; 043import org.apache.hadoop.hbase.client.Put; 044import org.apache.hadoop.hbase.client.Table; 045import org.apache.hadoop.hbase.filter.ParseFilter; 046import org.apache.hadoop.hbase.security.UserProvider; 047import org.apache.hadoop.hbase.test.MetricsAssertHelper; 048import org.apache.hadoop.hbase.testclassification.ClientTests; 049import org.apache.hadoop.hbase.testclassification.LargeTests; 050import org.apache.hadoop.hbase.thrift.generated.BatchMutation; 051import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor; 052import org.apache.hadoop.hbase.thrift.generated.Hbase; 053import org.apache.hadoop.hbase.thrift.generated.IOError; 054import org.apache.hadoop.hbase.thrift.generated.Mutation; 055import org.apache.hadoop.hbase.thrift.generated.TAppend; 056import org.apache.hadoop.hbase.thrift.generated.TCell; 057import org.apache.hadoop.hbase.thrift.generated.TIncrement; 058import org.apache.hadoop.hbase.thrift.generated.TRegionInfo; 059import org.apache.hadoop.hbase.thrift.generated.TRowResult; 060import org.apache.hadoop.hbase.thrift.generated.TScan; 061import org.apache.hadoop.hbase.util.Bytes; 062import org.apache.hadoop.hbase.util.TableDescriptorChecker; 063import org.apache.hadoop.hbase.util.Threads; 064import org.junit.AfterClass; 065import org.junit.BeforeClass; 066import org.junit.ClassRule; 067import org.junit.Rule; 068import org.junit.Test; 069import org.junit.experimental.categories.Category; 070import org.junit.rules.TestName; 071import org.slf4j.Logger; 072import org.slf4j.LoggerFactory; 073 074/** 075 * Unit testing for ThriftServerRunner.HBaseServiceHandler, a part of the 076 * org.apache.hadoop.hbase.thrift package. 077 */ 078@Category({ClientTests.class, LargeTests.class}) 079public class TestThriftServer { 080 081 @ClassRule 082 public static final HBaseClassTestRule CLASS_RULE = 083 HBaseClassTestRule.forClass(TestThriftServer.class); 084 085 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); 086 private static final Logger LOG = LoggerFactory.getLogger(TestThriftServer.class); 087 private static final MetricsAssertHelper metricsHelper = CompatibilityFactory 088 .getInstance(MetricsAssertHelper.class); 089 protected static final int MAXVERSIONS = 3; 090 091 private static ByteBuffer asByteBuffer(String i) { 092 return ByteBuffer.wrap(Bytes.toBytes(i)); 093 } 094 private static ByteBuffer asByteBuffer(long l) { 095 return ByteBuffer.wrap(Bytes.toBytes(l)); 096 } 097 098 // Static names for tables, columns, rows, and values 099 private static ByteBuffer tableAname = asByteBuffer("tableA"); 100 private static ByteBuffer tableBname = asByteBuffer("tableB"); 101 private static ByteBuffer columnAname = asByteBuffer("columnA:"); 102 private static ByteBuffer columnAAname = asByteBuffer("columnA:A"); 103 private static ByteBuffer columnBname = asByteBuffer("columnB:"); 104 private static ByteBuffer rowAname = asByteBuffer("rowA"); 105 private static ByteBuffer rowBname = asByteBuffer("rowB"); 106 private static ByteBuffer valueAname = asByteBuffer("valueA"); 107 private static ByteBuffer valueBname = asByteBuffer("valueB"); 108 private static ByteBuffer valueCname = asByteBuffer("valueC"); 109 private static ByteBuffer valueDname = asByteBuffer("valueD"); 110 private static ByteBuffer valueEname = asByteBuffer(100l); 111 112 @Rule 113 public TestName name = new TestName(); 114 115 @BeforeClass 116 public static void beforeClass() throws Exception { 117 UTIL.getConfiguration().setBoolean(COALESCE_INC_KEY, true); 118 UTIL.getConfiguration().setBoolean(TableDescriptorChecker.TABLE_SANITY_CHECKS, false); 119 UTIL.getConfiguration().setInt("hbase.client.retries.number", 3); 120 UTIL.startMiniCluster(); 121 } 122 123 @AfterClass 124 public static void afterClass() throws Exception { 125 UTIL.shutdownMiniCluster(); 126 } 127 128 /** 129 * Runs all of the tests under a single JUnit test method. We 130 * consolidate all testing to one method because HBaseClusterTestCase 131 * is prone to OutOfMemoryExceptions when there are three or more 132 * JUnit test methods. 133 * 134 * @throws Exception 135 */ 136 @Test 137 public void testAll() throws Exception { 138 // Run all tests 139 doTestTableCreateDrop(); 140 doTestThriftMetrics(); 141 doTestTableMutations(); 142 doTestTableTimestampsAndColumns(); 143 doTestTableScanners(); 144 doTestGetTableRegions(); 145 doTestFilterRegistration(); 146 doTestGetRegionInfo(); 147 doTestIncrements(); 148 doTestAppend(); 149 doTestCheckAndPut(); 150 } 151 152 /** 153 * Tests for creating, enabling, disabling, and deleting tables. Also 154 * tests that creating a table with an invalid column name yields an 155 * IllegalArgument exception. 156 * 157 * @throws Exception 158 */ 159 public void doTestTableCreateDrop() throws Exception { 160 ThriftHBaseServiceHandler handler = 161 new ThriftHBaseServiceHandler(UTIL.getConfiguration(), 162 UserProvider.instantiate(UTIL.getConfiguration())); 163 doTestTableCreateDrop(handler); 164 } 165 166 public static void doTestTableCreateDrop(Hbase.Iface handler) throws Exception { 167 createTestTables(handler); 168 dropTestTables(handler); 169 } 170 171 public static final class MySlowHBaseHandler extends ThriftHBaseServiceHandler 172 implements Hbase.Iface { 173 174 protected MySlowHBaseHandler(Configuration c) 175 throws IOException { 176 super(c, UserProvider.instantiate(c)); 177 } 178 179 @Override 180 public List<ByteBuffer> getTableNames() throws IOError { 181 Threads.sleepWithoutInterrupt(3000); 182 return super.getTableNames(); 183 } 184 } 185 186 /** 187 * TODO: These counts are supposed to be zero but sometimes they are not, they are equal to the 188 * passed in maybe. Investigate why. My guess is they are set by the test that runs just 189 * previous to this one. Sometimes they are cleared. Sometimes not. 190 * @param name 191 * @param maybe 192 * @param metrics 193 * @return 194 */ 195 private int getCurrentCount(final String name, final int maybe, final ThriftMetrics metrics) { 196 int currentCount = 0; 197 try { 198 metricsHelper.assertCounter(name, maybe, metrics.getSource()); 199 LOG.info("Shouldn't this be null? name=" + name + ", equals=" + maybe); 200 currentCount = maybe; 201 } catch (AssertionError e) { 202 // Ignore 203 } 204 return currentCount; 205 } 206 207 /** 208 * Tests if the metrics for thrift handler work correctly 209 */ 210 public void doTestThriftMetrics() throws Exception { 211 LOG.info("START doTestThriftMetrics"); 212 Configuration conf = UTIL.getConfiguration(); 213 ThriftMetrics metrics = getMetrics(conf); 214 Hbase.Iface handler = getHandlerForMetricsTest(metrics, conf); 215 int currentCountCreateTable = getCurrentCount("createTable_num_ops", 2, metrics); 216 int currentCountDeleteTable = getCurrentCount("deleteTable_num_ops", 2, metrics); 217 int currentCountDisableTable = getCurrentCount("disableTable_num_ops", 2, metrics); 218 createTestTables(handler); 219 dropTestTables(handler);; 220 metricsHelper.assertCounter("createTable_num_ops", currentCountCreateTable + 2, 221 metrics.getSource()); 222 metricsHelper.assertCounter("deleteTable_num_ops", currentCountDeleteTable + 2, 223 metrics.getSource()); 224 metricsHelper.assertCounter("disableTable_num_ops", currentCountDisableTable + 2, 225 metrics.getSource()); 226 handler.getTableNames(); // This will have an artificial delay. 227 228 // 3 to 6 seconds (to account for potential slowness), measured in nanoseconds 229 try { 230 metricsHelper.assertGaugeGt("getTableNames_avg_time", 3L * 1000 * 1000 * 1000, metrics.getSource()); 231 metricsHelper.assertGaugeLt("getTableNames_avg_time",6L * 1000 * 1000 * 1000, metrics.getSource()); 232 } catch (AssertionError e) { 233 LOG.info("Fix me! Why does this happen? A concurrent cluster running?", e); 234 } 235 } 236 237 private static Hbase.Iface getHandlerForMetricsTest(ThriftMetrics metrics, Configuration conf) 238 throws Exception { 239 Hbase.Iface handler = new MySlowHBaseHandler(conf); 240 return HbaseHandlerMetricsProxy.newInstance((ThriftHBaseServiceHandler)handler, metrics, conf); 241 } 242 243 private static ThriftMetrics getMetrics(Configuration conf) throws Exception { 244 return new ThriftMetrics( conf, ThriftMetrics.ThriftServerType.ONE); 245 } 246 247 248 public static void createTestTables(Hbase.Iface handler) throws Exception { 249 // Create/enable/disable/delete tables, ensure methods act correctly 250 assertEquals(0, handler.getTableNames().size()); 251 handler.createTable(tableAname, getColumnDescriptors()); 252 assertEquals(1, handler.getTableNames().size()); 253 assertEquals(2, handler.getColumnDescriptors(tableAname).size()); 254 assertTrue(handler.isTableEnabled(tableAname)); 255 handler.createTable(tableBname, getColumnDescriptors()); 256 assertEquals(2, handler.getTableNames().size()); 257 } 258 259 public static void checkTableList(Hbase.Iface handler) throws Exception { 260 assertTrue(handler.getTableNames().contains(tableAname)); 261 } 262 263 public static void dropTestTables(Hbase.Iface handler) throws Exception { 264 handler.disableTable(tableBname); 265 assertFalse(handler.isTableEnabled(tableBname)); 266 handler.deleteTable(tableBname); 267 assertEquals(1, handler.getTableNames().size()); 268 handler.disableTable(tableAname); 269 assertFalse(handler.isTableEnabled(tableAname)); 270 /* TODO Reenable. 271 assertFalse(handler.isTableEnabled(tableAname)); 272 handler.enableTable(tableAname); 273 assertTrue(handler.isTableEnabled(tableAname)); 274 handler.disableTable(tableAname);*/ 275 handler.deleteTable(tableAname); 276 assertEquals(0, handler.getTableNames().size()); 277 } 278 279 public void doTestIncrements() throws Exception { 280 ThriftHBaseServiceHandler handler = 281 new ThriftHBaseServiceHandler(UTIL.getConfiguration(), 282 UserProvider.instantiate(UTIL.getConfiguration())); 283 createTestTables(handler); 284 doTestIncrements(handler); 285 dropTestTables(handler); 286 } 287 288 public static void doTestIncrements(ThriftHBaseServiceHandler handler) throws Exception { 289 List<Mutation> mutations = new ArrayList<>(1); 290 mutations.add(new Mutation(false, columnAAname, valueEname, true)); 291 mutations.add(new Mutation(false, columnAname, valueEname, true)); 292 handler.mutateRow(tableAname, rowAname, mutations, null); 293 handler.mutateRow(tableAname, rowBname, mutations, null); 294 295 List<TIncrement> increments = new ArrayList<>(3); 296 increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7)); 297 increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7)); 298 increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7)); 299 300 int numIncrements = 60000; 301 for (int i = 0; i < numIncrements; i++) { 302 handler.increment(new TIncrement(tableAname, rowAname, columnAname, 2)); 303 handler.incrementRows(increments); 304 } 305 306 Thread.sleep(1000); 307 long lv = handler.get(tableAname, rowAname, columnAname, null).get(0).value.getLong(); 308 // Wait on all increments being flushed 309 while (handler.coalescer.getQueueSize() != 0) Threads.sleep(10); 310 assertEquals((100 + (2 * numIncrements)), lv ); 311 312 313 lv = handler.get(tableAname, rowBname, columnAAname, null).get(0).value.getLong(); 314 assertEquals((100 + (3 * 7 * numIncrements)), lv); 315 316 assertTrue(handler.coalescer.getSuccessfulCoalescings() > 0); 317 318 } 319 320 /** 321 * Tests adding a series of Mutations and BatchMutations, including a 322 * delete mutation. Also tests data retrieval, and getting back multiple 323 * versions. 324 * 325 * @throws Exception 326 */ 327 public void doTestTableMutations() throws Exception { 328 ThriftHBaseServiceHandler handler = 329 new ThriftHBaseServiceHandler(UTIL.getConfiguration(), 330 UserProvider.instantiate(UTIL.getConfiguration())); 331 doTestTableMutations(handler); 332 } 333 334 public static void doTestTableMutations(Hbase.Iface handler) throws Exception { 335 // Setup 336 handler.createTable(tableAname, getColumnDescriptors()); 337 338 // Apply a few Mutations to rowA 339 // mutations.add(new Mutation(false, columnAname, valueAname)); 340 // mutations.add(new Mutation(false, columnBname, valueBname)); 341 handler.mutateRow(tableAname, rowAname, getMutations(), null); 342 343 // Assert that the changes were made 344 assertEquals(valueAname, 345 handler.get(tableAname, rowAname, columnAname, null).get(0).value); 346 TRowResult rowResult1 = handler.getRow(tableAname, rowAname, null).get(0); 347 assertEquals(rowAname, rowResult1.row); 348 assertEquals(valueBname, 349 rowResult1.columns.get(columnBname).value); 350 351 // Apply a few BatchMutations for rowA and rowB 352 // rowAmutations.add(new Mutation(true, columnAname, null)); 353 // rowAmutations.add(new Mutation(false, columnBname, valueCname)); 354 // batchMutations.add(new BatchMutation(rowAname, rowAmutations)); 355 // Mutations to rowB 356 // rowBmutations.add(new Mutation(false, columnAname, valueCname)); 357 // rowBmutations.add(new Mutation(false, columnBname, valueDname)); 358 // batchMutations.add(new BatchMutation(rowBname, rowBmutations)); 359 handler.mutateRows(tableAname, getBatchMutations(), null); 360 361 // Assert that changes were made to rowA 362 List<TCell> cells = handler.get(tableAname, rowAname, columnAname, null); 363 assertFalse(cells.size() > 0); 364 assertEquals(valueCname, handler.get(tableAname, rowAname, columnBname, null).get(0).value); 365 List<TCell> versions = handler.getVer(tableAname, rowAname, columnBname, MAXVERSIONS, null); 366 assertEquals(valueCname, versions.get(0).value); 367 assertEquals(valueBname, versions.get(1).value); 368 369 // Assert that changes were made to rowB 370 TRowResult rowResult2 = handler.getRow(tableAname, rowBname, null).get(0); 371 assertEquals(rowBname, rowResult2.row); 372 assertEquals(valueCname, rowResult2.columns.get(columnAname).value); 373 assertEquals(valueDname, rowResult2.columns.get(columnBname).value); 374 375 // Apply some deletes 376 handler.deleteAll(tableAname, rowAname, columnBname, null); 377 handler.deleteAllRow(tableAname, rowBname, null); 378 379 // Assert that the deletes were applied 380 int size = handler.get(tableAname, rowAname, columnBname, null).size(); 381 assertEquals(0, size); 382 size = handler.getRow(tableAname, rowBname, null).size(); 383 assertEquals(0, size); 384 385 // Try null mutation 386 List<Mutation> mutations = new ArrayList<>(1); 387 mutations.add(new Mutation(false, columnAname, null, true)); 388 handler.mutateRow(tableAname, rowAname, mutations, null); 389 TRowResult rowResult3 = handler.getRow(tableAname, rowAname, null).get(0); 390 assertEquals(rowAname, rowResult3.row); 391 assertEquals(0, rowResult3.columns.get(columnAname).value.remaining()); 392 393 // Teardown 394 handler.disableTable(tableAname); 395 handler.deleteTable(tableAname); 396 } 397 398 /** 399 * Similar to testTableMutations(), except Mutations are applied with 400 * specific timestamps and data retrieval uses these timestamps to 401 * extract specific versions of data. 402 * 403 * @throws Exception 404 */ 405 public void doTestTableTimestampsAndColumns() throws Exception { 406 // Setup 407 ThriftHBaseServiceHandler handler = 408 new ThriftHBaseServiceHandler(UTIL.getConfiguration(), 409 UserProvider.instantiate(UTIL.getConfiguration())); 410 handler.createTable(tableAname, getColumnDescriptors()); 411 412 // Apply timestamped Mutations to rowA 413 long time1 = System.currentTimeMillis(); 414 handler.mutateRowTs(tableAname, rowAname, getMutations(), time1, null); 415 416 Thread.sleep(1000); 417 418 // Apply timestamped BatchMutations for rowA and rowB 419 long time2 = System.currentTimeMillis(); 420 handler.mutateRowsTs(tableAname, getBatchMutations(), time2, null); 421 422 // Apply an overlapping timestamped mutation to rowB 423 handler.mutateRowTs(tableAname, rowBname, getMutations(), time2, null); 424 425 // the getVerTs is [inf, ts) so you need to increment one. 426 time1 += 1; 427 time2 += 2; 428 429 // Assert that the timestamp-related methods retrieve the correct data 430 assertEquals(2, handler.getVerTs(tableAname, rowAname, columnBname, time2, 431 MAXVERSIONS, null).size()); 432 assertEquals(1, handler.getVerTs(tableAname, rowAname, columnBname, time1, 433 MAXVERSIONS, null).size()); 434 435 TRowResult rowResult1 = handler.getRowTs(tableAname, rowAname, time1, null).get(0); 436 TRowResult rowResult2 = handler.getRowTs(tableAname, rowAname, time2, null).get(0); 437 // columnA was completely deleted 438 //assertTrue(Bytes.equals(rowResult1.columns.get(columnAname).value, valueAname)); 439 assertEquals(rowResult1.columns.get(columnBname).value, valueBname); 440 assertEquals(rowResult2.columns.get(columnBname).value, valueCname); 441 442 // ColumnAname has been deleted, and will never be visible even with a getRowTs() 443 assertFalse(rowResult2.columns.containsKey(columnAname)); 444 445 List<ByteBuffer> columns = new ArrayList<>(1); 446 columns.add(columnBname); 447 448 rowResult1 = handler.getRowWithColumns(tableAname, rowAname, columns, null).get(0); 449 assertEquals(rowResult1.columns.get(columnBname).value, valueCname); 450 assertFalse(rowResult1.columns.containsKey(columnAname)); 451 452 rowResult1 = handler.getRowWithColumnsTs(tableAname, rowAname, columns, time1, null).get(0); 453 assertEquals(rowResult1.columns.get(columnBname).value, valueBname); 454 assertFalse(rowResult1.columns.containsKey(columnAname)); 455 456 // Apply some timestamped deletes 457 // this actually deletes _everything_. 458 // nukes everything in columnB: forever. 459 handler.deleteAllTs(tableAname, rowAname, columnBname, time1, null); 460 handler.deleteAllRowTs(tableAname, rowBname, time2, null); 461 462 // Assert that the timestamp-related methods retrieve the correct data 463 int size = handler.getVerTs(tableAname, rowAname, columnBname, time1, MAXVERSIONS, null).size(); 464 assertEquals(0, size); 465 466 size = handler.getVerTs(tableAname, rowAname, columnBname, time2, MAXVERSIONS, null).size(); 467 assertEquals(1, size); 468 469 // should be available.... 470 assertEquals(handler.get(tableAname, rowAname, columnBname, null).get(0).value, valueCname); 471 472 assertEquals(0, handler.getRow(tableAname, rowBname, null).size()); 473 474 // Teardown 475 handler.disableTable(tableAname); 476 handler.deleteTable(tableAname); 477 } 478 479 /** 480 * Tests the four different scanner-opening methods (with and without 481 * a stoprow, with and without a timestamp). 482 * 483 * @throws Exception 484 */ 485 public void doTestTableScanners() throws Exception { 486 // Setup 487 ThriftHBaseServiceHandler handler = 488 new ThriftHBaseServiceHandler(UTIL.getConfiguration(), 489 UserProvider.instantiate(UTIL.getConfiguration())); 490 handler.createTable(tableAname, getColumnDescriptors()); 491 492 // Apply timestamped Mutations to rowA 493 long time1 = System.currentTimeMillis(); 494 handler.mutateRowTs(tableAname, rowAname, getMutations(), time1, null); 495 496 // Sleep to assure that 'time1' and 'time2' will be different even with a 497 // coarse grained system timer. 498 Thread.sleep(1000); 499 500 // Apply timestamped BatchMutations for rowA and rowB 501 long time2 = System.currentTimeMillis(); 502 handler.mutateRowsTs(tableAname, getBatchMutations(), time2, null); 503 504 time1 += 1; 505 506 // Test a scanner on all rows and all columns, no timestamp 507 int scanner1 = handler.scannerOpen(tableAname, rowAname, getColumnList(true, true), null); 508 TRowResult rowResult1a = handler.scannerGet(scanner1).get(0); 509 assertEquals(rowResult1a.row, rowAname); 510 // This used to be '1'. I don't know why when we are asking for two columns 511 // and when the mutations above would seem to add two columns to the row. 512 // -- St.Ack 05/12/2009 513 assertEquals(1, rowResult1a.columns.size()); 514 assertEquals(rowResult1a.columns.get(columnBname).value, valueCname); 515 516 TRowResult rowResult1b = handler.scannerGet(scanner1).get(0); 517 assertEquals(rowResult1b.row, rowBname); 518 assertEquals(2, rowResult1b.columns.size()); 519 assertEquals(rowResult1b.columns.get(columnAname).value, valueCname); 520 assertEquals(rowResult1b.columns.get(columnBname).value, valueDname); 521 closeScanner(scanner1, handler); 522 523 // Test a scanner on all rows and all columns, with timestamp 524 int scanner2 = handler.scannerOpenTs(tableAname, rowAname, getColumnList(true, true), time1, null); 525 TRowResult rowResult2a = handler.scannerGet(scanner2).get(0); 526 assertEquals(1, rowResult2a.columns.size()); 527 // column A deleted, does not exist. 528 //assertTrue(Bytes.equals(rowResult2a.columns.get(columnAname).value, valueAname)); 529 assertEquals(rowResult2a.columns.get(columnBname).value, valueBname); 530 closeScanner(scanner2, handler); 531 532 // Test a scanner on the first row and first column only, no timestamp 533 int scanner3 = handler.scannerOpenWithStop(tableAname, rowAname, rowBname, 534 getColumnList(true, false), null); 535 closeScanner(scanner3, handler); 536 537 // Test a scanner on the first row and second column only, with timestamp 538 int scanner4 = handler.scannerOpenWithStopTs(tableAname, rowAname, rowBname, 539 getColumnList(false, true), time1, null); 540 TRowResult rowResult4a = handler.scannerGet(scanner4).get(0); 541 assertEquals(1, rowResult4a.columns.size()); 542 assertEquals(rowResult4a.columns.get(columnBname).value, valueBname); 543 544 // Test scanner using a TScan object once with sortColumns False and once with sortColumns true 545 TScan scanNoSortColumns = new TScan(); 546 scanNoSortColumns.setStartRow(rowAname); 547 scanNoSortColumns.setStopRow(rowBname); 548 549 int scanner5 = handler.scannerOpenWithScan(tableAname , scanNoSortColumns, null); 550 TRowResult rowResult5 = handler.scannerGet(scanner5).get(0); 551 assertEquals(1, rowResult5.columns.size()); 552 assertEquals(rowResult5.columns.get(columnBname).value, valueCname); 553 554 TScan scanSortColumns = new TScan(); 555 scanSortColumns.setStartRow(rowAname); 556 scanSortColumns.setStopRow(rowBname); 557 scanSortColumns = scanSortColumns.setSortColumns(true); 558 559 int scanner6 = handler.scannerOpenWithScan(tableAname ,scanSortColumns, null); 560 TRowResult rowResult6 = handler.scannerGet(scanner6).get(0); 561 assertEquals(1, rowResult6.sortedColumns.size()); 562 assertEquals(rowResult6.sortedColumns.get(0).getCell().value, valueCname); 563 564 List<Mutation> rowBmutations = new ArrayList<>(20); 565 for (int i = 0; i < 20; i++) { 566 rowBmutations.add(new Mutation(false, asByteBuffer("columnA:" + i), valueCname, true)); 567 } 568 ByteBuffer rowC = asByteBuffer("rowC"); 569 handler.mutateRow(tableAname, rowC, rowBmutations, null); 570 571 TScan scanSortMultiColumns = new TScan(); 572 scanSortMultiColumns.setStartRow(rowC); 573 scanSortMultiColumns = scanSortMultiColumns.setSortColumns(true); 574 int scanner7 = handler.scannerOpenWithScan(tableAname, scanSortMultiColumns, null); 575 TRowResult rowResult7 = handler.scannerGet(scanner7).get(0); 576 577 ByteBuffer smallerColumn = asByteBuffer("columnA:"); 578 for (int i = 0; i < 20; i++) { 579 ByteBuffer currentColumn = rowResult7.sortedColumns.get(i).columnName; 580 assertTrue(Bytes.compareTo(smallerColumn.array(), currentColumn.array()) < 0); 581 smallerColumn = currentColumn; 582 } 583 584 TScan reversedScan = new TScan(); 585 reversedScan.setReversed(true); 586 reversedScan.setStartRow(rowBname); 587 reversedScan.setStopRow(rowAname); 588 589 int scanner8 = handler.scannerOpenWithScan(tableAname , reversedScan, null); 590 List<TRowResult> results = handler.scannerGet(scanner8); 591 handler.scannerClose(scanner8); 592 assertEquals(1, results.size()); 593 assertEquals(ByteBuffer.wrap(results.get(0).getRow()), rowBname); 594 595 // Teardown 596 handler.disableTable(tableAname); 597 handler.deleteTable(tableAname); 598 } 599 600 /** 601 * For HBASE-2556 602 * Tests for GetTableRegions 603 * 604 * @throws Exception 605 */ 606 public void doTestGetTableRegions() throws Exception { 607 ThriftHBaseServiceHandler handler = 608 new ThriftHBaseServiceHandler(UTIL.getConfiguration(), 609 UserProvider.instantiate(UTIL.getConfiguration())); 610 doTestGetTableRegions(handler); 611 } 612 613 public static void doTestGetTableRegions(Hbase.Iface handler) 614 throws Exception { 615 assertEquals(0, handler.getTableNames().size()); 616 handler.createTable(tableAname, getColumnDescriptors()); 617 assertEquals(1, handler.getTableNames().size()); 618 List<TRegionInfo> regions = handler.getTableRegions(tableAname); 619 int regionCount = regions.size(); 620 assertEquals("empty table should have only 1 region, " + 621 "but found " + regionCount, 1, regionCount); 622 LOG.info("Region found:" + regions.get(0)); 623 handler.disableTable(tableAname); 624 handler.deleteTable(tableAname); 625 regionCount = handler.getTableRegions(tableAname).size(); 626 assertEquals("non-existing table should have 0 region, " + 627 "but found " + regionCount, 0, regionCount); 628 } 629 630 public void doTestFilterRegistration() throws Exception { 631 Configuration conf = UTIL.getConfiguration(); 632 633 conf.set("hbase.thrift.filters", "MyFilter:filterclass"); 634 635 ThriftServer.registerFilters(conf); 636 637 Map<String, String> registeredFilters = ParseFilter.getAllFilters(); 638 639 assertEquals("filterclass", registeredFilters.get("MyFilter")); 640 } 641 642 public void doTestGetRegionInfo() throws Exception { 643 ThriftHBaseServiceHandler handler = 644 new ThriftHBaseServiceHandler(UTIL.getConfiguration(), 645 UserProvider.instantiate(UTIL.getConfiguration())); 646 doTestGetRegionInfo(handler); 647 } 648 649 public static void doTestGetRegionInfo(Hbase.Iface handler) throws Exception { 650 // Create tableA and add two columns to rowA 651 handler.createTable(tableAname, getColumnDescriptors()); 652 try { 653 handler.mutateRow(tableAname, rowAname, getMutations(), null); 654 byte[] searchRow = HRegionInfo.createRegionName( 655 TableName.valueOf(tableAname.array()), rowAname.array(), 656 HConstants.NINES, false); 657 TRegionInfo regionInfo = handler.getRegionInfo(ByteBuffer.wrap(searchRow)); 658 assertTrue(Bytes.toStringBinary(regionInfo.getName()).startsWith( 659 Bytes.toStringBinary(tableAname))); 660 } finally { 661 handler.disableTable(tableAname); 662 handler.deleteTable(tableAname); 663 } 664 } 665 666 /** 667 * Appends the value to a cell and checks that the cell value is updated properly. 668 * 669 * @throws Exception 670 */ 671 public static void doTestAppend() throws Exception { 672 ThriftHBaseServiceHandler handler = 673 new ThriftHBaseServiceHandler(UTIL.getConfiguration(), 674 UserProvider.instantiate(UTIL.getConfiguration())); 675 handler.createTable(tableAname, getColumnDescriptors()); 676 try { 677 List<Mutation> mutations = new ArrayList<>(1); 678 mutations.add(new Mutation(false, columnAname, valueAname, true)); 679 handler.mutateRow(tableAname, rowAname, mutations, null); 680 681 List<ByteBuffer> columnList = new ArrayList<>(1); 682 columnList.add(columnAname); 683 List<ByteBuffer> valueList = new ArrayList<>(1); 684 valueList.add(valueBname); 685 686 TAppend append = new TAppend(tableAname, rowAname, columnList, valueList); 687 handler.append(append); 688 689 TRowResult rowResult = handler.getRow(tableAname, rowAname, null).get(0); 690 assertEquals(rowAname, rowResult.row); 691 assertArrayEquals(Bytes.add(valueAname.array(), valueBname.array()), 692 rowResult.columns.get(columnAname).value.array()); 693 } finally { 694 handler.disableTable(tableAname); 695 handler.deleteTable(tableAname); 696 } 697 } 698 699 /** 700 * Check that checkAndPut fails if the cell does not exist, then put in the cell, then check that 701 * the checkAndPut succeeds. 702 * 703 * @throws Exception 704 */ 705 public static void doTestCheckAndPut() throws Exception { 706 ThriftHBaseServiceHandler handler = 707 new ThriftHBaseServiceHandler(UTIL.getConfiguration(), 708 UserProvider.instantiate(UTIL.getConfiguration())); 709 handler.createTable(tableAname, getColumnDescriptors()); 710 try { 711 List<Mutation> mutations = new ArrayList<>(1); 712 mutations.add(new Mutation(false, columnAname, valueAname, true)); 713 Mutation putB = (new Mutation(false, columnBname, valueBname, true)); 714 715 assertFalse(handler.checkAndPut(tableAname, rowAname, columnAname, valueAname, putB, null)); 716 717 handler.mutateRow(tableAname, rowAname, mutations, null); 718 719 assertTrue(handler.checkAndPut(tableAname, rowAname, columnAname, valueAname, putB, null)); 720 721 TRowResult rowResult = handler.getRow(tableAname, rowAname, null).get(0); 722 assertEquals(rowAname, rowResult.row); 723 assertEquals(valueBname, rowResult.columns.get(columnBname).value); 724 } finally { 725 handler.disableTable(tableAname); 726 handler.deleteTable(tableAname); 727 } 728 } 729 730 @Test 731 public void testMetricsWithException() throws Exception { 732 String rowkey = "row1"; 733 String family = "f"; 734 String col = "c"; 735 // create a table which will throw exceptions for requests 736 final TableName tableName = TableName.valueOf(name.getMethodName()); 737 HTableDescriptor tableDesc = new HTableDescriptor(tableName); 738 tableDesc.addCoprocessor(ErrorThrowingGetObserver.class.getName()); 739 tableDesc.addFamily(new HColumnDescriptor(family)); 740 741 Table table = UTIL.createTable(tableDesc, null); 742 long now = System.currentTimeMillis(); 743 table.put(new Put(Bytes.toBytes(rowkey)) 744 .addColumn(Bytes.toBytes(family), Bytes.toBytes(col), now, Bytes.toBytes("val1"))); 745 746 Configuration conf = UTIL.getConfiguration(); 747 ThriftMetrics metrics = getMetrics(conf); 748 ThriftHBaseServiceHandler hbaseHandler = 749 new ThriftHBaseServiceHandler(UTIL.getConfiguration(), 750 UserProvider.instantiate(UTIL.getConfiguration())); 751 Hbase.Iface handler = HbaseHandlerMetricsProxy.newInstance(hbaseHandler, metrics, conf); 752 753 ByteBuffer tTableName = asByteBuffer(tableName.getNameAsString()); 754 755 // check metrics increment with a successful get 756 long preGetCounter = metricsHelper.checkCounterExists("getRow_num_ops", metrics.getSource()) ? 757 metricsHelper.getCounter("getRow_num_ops", metrics.getSource()) : 758 0; 759 List<TRowResult> tRowResult = handler.getRow(tTableName, asByteBuffer(rowkey), null); 760 assertEquals(1, tRowResult.size()); 761 TRowResult tResult = tRowResult.get(0); 762 763 TCell expectedColumnValue = new TCell(asByteBuffer("val1"), now); 764 765 assertArrayEquals(Bytes.toBytes(rowkey), tResult.getRow()); 766 Collection<TCell> returnedColumnValues = tResult.getColumns().values(); 767 assertEquals(1, returnedColumnValues.size()); 768 assertEquals(expectedColumnValue, returnedColumnValues.iterator().next()); 769 770 metricsHelper.assertCounter("getRow_num_ops", preGetCounter + 1, metrics.getSource()); 771 772 // check metrics increment when the get throws each exception type 773 for (ErrorThrowingGetObserver.ErrorType type : ErrorThrowingGetObserver.ErrorType.values()) { 774 testExceptionType(handler, metrics, tTableName, rowkey, type); 775 } 776 } 777 778 private void testExceptionType(Hbase.Iface handler, ThriftMetrics metrics, 779 ByteBuffer tTableName, String rowkey, 780 ErrorThrowingGetObserver.ErrorType errorType) throws Exception { 781 long preGetCounter = metricsHelper.getCounter("getRow_num_ops", metrics.getSource()); 782 String exceptionKey = errorType.getMetricName(); 783 long preExceptionCounter = metricsHelper.checkCounterExists(exceptionKey, metrics.getSource()) ? 784 metricsHelper.getCounter(exceptionKey, metrics.getSource()) : 785 0; 786 Map<ByteBuffer, ByteBuffer> attributes = new HashMap<>(); 787 attributes.put(asByteBuffer(ErrorThrowingGetObserver.SHOULD_ERROR_ATTRIBUTE), 788 asByteBuffer(errorType.name())); 789 try { 790 List<TRowResult> tRowResult = handler.getRow(tTableName, asByteBuffer(rowkey), attributes); 791 fail("Get with error attribute should have thrown an exception"); 792 } catch (IOError e) { 793 LOG.info("Received exception: ", e); 794 metricsHelper.assertCounter("getRow_num_ops", preGetCounter + 1, metrics.getSource()); 795 metricsHelper.assertCounter(exceptionKey, preExceptionCounter + 1, metrics.getSource()); 796 } 797 } 798 799 /** 800 * 801 * @return a List of ColumnDescriptors for use in creating a table. Has one 802 * default ColumnDescriptor and one ColumnDescriptor with fewer versions 803 */ 804 private static List<ColumnDescriptor> getColumnDescriptors() { 805 ArrayList<ColumnDescriptor> cDescriptors = new ArrayList<>(2); 806 807 // A default ColumnDescriptor 808 ColumnDescriptor cDescA = new ColumnDescriptor(); 809 cDescA.name = columnAname; 810 cDescriptors.add(cDescA); 811 812 // A slightly customized ColumnDescriptor (only 2 versions) 813 ColumnDescriptor cDescB = new ColumnDescriptor(columnBname, 2, "NONE", 814 false, "NONE", 0, 0, false, -1); 815 cDescriptors.add(cDescB); 816 817 return cDescriptors; 818 } 819 820 /** 821 * 822 * @param includeA whether or not to include columnA 823 * @param includeB whether or not to include columnB 824 * @return a List of column names for use in retrieving a scanner 825 */ 826 private List<ByteBuffer> getColumnList(boolean includeA, boolean includeB) { 827 List<ByteBuffer> columnList = new ArrayList<>(); 828 if (includeA) columnList.add(columnAname); 829 if (includeB) columnList.add(columnBname); 830 return columnList; 831 } 832 833 /** 834 * 835 * @return a List of Mutations for a row, with columnA having valueA 836 * and columnB having valueB 837 */ 838 private static List<Mutation> getMutations() { 839 List<Mutation> mutations = new ArrayList<>(2); 840 mutations.add(new Mutation(false, columnAname, valueAname, true)); 841 mutations.add(new Mutation(false, columnBname, valueBname, true)); 842 return mutations; 843 } 844 845 /** 846 * 847 * @return a List of BatchMutations with the following effects: 848 * (rowA, columnA): delete 849 * (rowA, columnB): place valueC 850 * (rowB, columnA): place valueC 851 * (rowB, columnB): place valueD 852 */ 853 private static List<BatchMutation> getBatchMutations() { 854 List<BatchMutation> batchMutations = new ArrayList<>(3); 855 856 // Mutations to rowA. You can't mix delete and put anymore. 857 List<Mutation> rowAmutations = new ArrayList<>(1); 858 rowAmutations.add(new Mutation(true, columnAname, null, true)); 859 batchMutations.add(new BatchMutation(rowAname, rowAmutations)); 860 861 rowAmutations = new ArrayList<>(1); 862 rowAmutations.add(new Mutation(false, columnBname, valueCname, true)); 863 batchMutations.add(new BatchMutation(rowAname, rowAmutations)); 864 865 // Mutations to rowB 866 List<Mutation> rowBmutations = new ArrayList<>(2); 867 rowBmutations.add(new Mutation(false, columnAname, valueCname, true)); 868 rowBmutations.add(new Mutation(false, columnBname, valueDname, true)); 869 batchMutations.add(new BatchMutation(rowBname, rowBmutations)); 870 871 return batchMutations; 872 } 873 874 /** 875 * Asserts that the passed scanner is exhausted, and then closes 876 * the scanner. 877 * 878 * @param scannerId the scanner to close 879 * @param handler the HBaseServiceHandler interfacing to HBase 880 * @throws Exception 881 */ 882 private void closeScanner( 883 int scannerId, ThriftHBaseServiceHandler handler) throws Exception { 884 handler.scannerGet(scannerId); 885 handler.scannerClose(scannerId); 886 } 887}