001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.client; 019 020import static org.junit.Assert.assertArrayEquals; 021import static org.junit.Assert.assertEquals; 022import static org.junit.Assert.assertFalse; 023import static org.junit.Assert.assertNotNull; 024import static org.junit.Assert.assertNull; 025import static org.junit.Assert.assertSame; 026import static org.junit.Assert.assertTrue; 027import static org.junit.Assert.fail; 028 029import java.io.IOException; 030import java.util.ArrayList; 031import java.util.Arrays; 032import java.util.EnumSet; 033import java.util.HashMap; 034import java.util.HashSet; 035import java.util.Iterator; 036import java.util.LinkedList; 037import java.util.List; 038import java.util.Map; 039import java.util.NavigableMap; 040import java.util.concurrent.Callable; 041import java.util.concurrent.ExecutorService; 042import java.util.concurrent.Executors; 043import java.util.concurrent.atomic.AtomicReference; 044import org.apache.commons.lang3.ArrayUtils; 045import org.apache.hadoop.conf.Configuration; 046import org.apache.hadoop.fs.Path; 047import org.apache.hadoop.hbase.Cell; 048import org.apache.hadoop.hbase.CellScanner; 049import org.apache.hadoop.hbase.CellUtil; 050import org.apache.hadoop.hbase.ClusterMetrics.Option; 051import org.apache.hadoop.hbase.CompareOperator; 052import org.apache.hadoop.hbase.DoNotRetryIOException; 053import org.apache.hadoop.hbase.HBaseClassTestRule; 054import org.apache.hadoop.hbase.HBaseTestingUtility; 055import org.apache.hadoop.hbase.HColumnDescriptor; 056import org.apache.hadoop.hbase.HConstants; 057import org.apache.hadoop.hbase.HRegionInfo; 058import org.apache.hadoop.hbase.HRegionLocation; 059import org.apache.hadoop.hbase.HTableDescriptor; 060import org.apache.hadoop.hbase.KeepDeletedCells; 061import org.apache.hadoop.hbase.KeyValue; 062import org.apache.hadoop.hbase.MiniHBaseCluster; 063import org.apache.hadoop.hbase.PrivateCellUtil; 064import org.apache.hadoop.hbase.RegionLocations; 065import org.apache.hadoop.hbase.ServerName; 066import org.apache.hadoop.hbase.TableName; 067import org.apache.hadoop.hbase.Waiter; 068import org.apache.hadoop.hbase.client.metrics.ScanMetrics; 069import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; 070import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint; 071import org.apache.hadoop.hbase.filter.BinaryComparator; 072import org.apache.hadoop.hbase.filter.Filter; 073import org.apache.hadoop.hbase.filter.FilterList; 074import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; 075import org.apache.hadoop.hbase.filter.InclusiveStopFilter; 076import org.apache.hadoop.hbase.filter.KeyOnlyFilter; 077import org.apache.hadoop.hbase.filter.LongComparator; 078import org.apache.hadoop.hbase.filter.PrefixFilter; 079import org.apache.hadoop.hbase.filter.QualifierFilter; 080import org.apache.hadoop.hbase.filter.RegexStringComparator; 081import org.apache.hadoop.hbase.filter.RowFilter; 082import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; 083import org.apache.hadoop.hbase.filter.SubstringComparator; 084import org.apache.hadoop.hbase.filter.ValueFilter; 085import org.apache.hadoop.hbase.filter.WhileMatchFilter; 086import org.apache.hadoop.hbase.io.TimeRange; 087import org.apache.hadoop.hbase.io.hfile.BlockCache; 088import org.apache.hadoop.hbase.io.hfile.CacheConfig; 089import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; 090import org.apache.hadoop.hbase.master.LoadBalancer; 091import org.apache.hadoop.hbase.protobuf.ProtobufUtil; 092import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto; 093import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType; 094import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService; 095import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest; 096import org.apache.hadoop.hbase.regionserver.HRegion; 097import org.apache.hadoop.hbase.regionserver.HRegionServer; 098import org.apache.hadoop.hbase.regionserver.HStore; 099import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; 100import org.apache.hadoop.hbase.testclassification.ClientTests; 101import org.apache.hadoop.hbase.testclassification.LargeTests; 102import org.apache.hadoop.hbase.util.Bytes; 103import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 104import org.apache.hadoop.hbase.util.FSUtils; 105import org.apache.hadoop.hbase.util.NonRepeatedEnvironmentEdge; 106import org.apache.hadoop.hbase.util.Pair; 107import org.junit.AfterClass; 108import org.junit.BeforeClass; 109import org.junit.ClassRule; 110import org.junit.Ignore; 111import org.junit.Rule; 112import org.junit.Test; 113import org.junit.experimental.categories.Category; 114import org.junit.rules.TestName; 115import org.slf4j.Logger; 116import org.slf4j.LoggerFactory; 117 118/** 119 * Run tests that use the HBase clients; {@link Table}. 120 * Sets up the HBase mini cluster once at start and runs through all client tests. 121 * Each creates a table named for the method and does its stuff against that. 122 */ 123@Category({LargeTests.class, ClientTests.class}) 124@SuppressWarnings ("deprecation") 125public class TestFromClientSide { 126 127 @ClassRule 128 public static final HBaseClassTestRule CLASS_RULE = 129 HBaseClassTestRule.forClass(TestFromClientSide.class); 130 131 // NOTE: Increment tests were moved to their own class, TestIncrementsFromClientSide. 132 private static final Logger LOG = LoggerFactory.getLogger(TestFromClientSide.class); 133 protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 134 private static byte [] ROW = Bytes.toBytes("testRow"); 135 private static byte [] FAMILY = Bytes.toBytes("testFamily"); 136 private static final byte[] INVALID_FAMILY = Bytes.toBytes("invalidTestFamily"); 137 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); 138 private static byte [] VALUE = Bytes.toBytes("testValue"); 139 protected static int SLAVES = 3; 140 141 @Rule 142 public TestName name = new TestName(); 143 144 protected static final void initialize(Class<?>... cps) throws Exception { 145 // Uncomment the following lines if more verbosity is needed for 146 // debugging (see HBASE-12285 for details). 147 // ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL); 148 // ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL); 149 // ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL); 150 // make sure that we do not get the same ts twice, see HBASE-19731 for more details. 151 EnvironmentEdgeManager.injectEdge(new NonRepeatedEnvironmentEdge()); 152 Configuration conf = TEST_UTIL.getConfiguration(); 153 conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, 154 Arrays.stream(cps).map(Class::getName).toArray(String[]::new)); 155 conf.setBoolean("hbase.table.sanity.checks", true); // enable for below tests 156 // We need more than one region server in this test 157 TEST_UTIL.startMiniCluster(SLAVES); 158 } 159 160 @BeforeClass 161 public static void setUpBeforeClass() throws Exception { 162 initialize(MultiRowMutationEndpoint.class); 163 } 164 165 @AfterClass 166 public static void tearDownAfterClass() throws Exception { 167 TEST_UTIL.shutdownMiniCluster(); 168 } 169 170 /** 171 * Test append result when there are duplicate rpc request. 172 */ 173 @Test 174 public void testDuplicateAppend() throws Exception { 175 HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(name.getMethodName()); 176 Map<String, String> kvs = new HashMap<>(); 177 kvs.put(HConnectionTestingUtility.SleepAtFirstRpcCall.SLEEP_TIME_CONF_KEY, "2000"); 178 hdt.addCoprocessor(HConnectionTestingUtility.SleepAtFirstRpcCall.class.getName(), null, 1, kvs); 179 TEST_UTIL.createTable(hdt, new byte[][] { ROW }).close(); 180 181 Configuration c = new Configuration(TEST_UTIL.getConfiguration()); 182 c.setInt(HConstants.HBASE_CLIENT_PAUSE, 50); 183 // Client will retry beacuse rpc timeout is small than the sleep time of first rpc call 184 c.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500); 185 186 try (Connection connection = ConnectionFactory.createConnection(c)) { 187 try (Table t = connection.getTable(TableName.valueOf(name.getMethodName()))) { 188 if (t instanceof HTable) { 189 HTable table = (HTable) t; 190 table.setOperationTimeout(3 * 1000); 191 192 Append append = new Append(ROW); 193 append.addColumn(HBaseTestingUtility.fam1, QUALIFIER, VALUE); 194 Result result = table.append(append); 195 196 // Verify expected result 197 Cell[] cells = result.rawCells(); 198 assertEquals(1, cells.length); 199 assertKey(cells[0], ROW, HBaseTestingUtility.fam1, QUALIFIER, VALUE); 200 201 // Verify expected result again 202 Result readResult = table.get(new Get(ROW)); 203 cells = readResult.rawCells(); 204 assertEquals(1, cells.length); 205 assertKey(cells[0], ROW, HBaseTestingUtility.fam1, QUALIFIER, VALUE); 206 } 207 } 208 } 209 } 210 211 /** 212 * Basic client side validation of HBASE-4536 213 */ 214 @Test 215 public void testKeepDeletedCells() throws Exception { 216 final TableName tableName = TableName.valueOf(name.getMethodName()); 217 final byte[] FAMILY = Bytes.toBytes("family"); 218 final byte[] C0 = Bytes.toBytes("c0"); 219 220 final byte[] T1 = Bytes.toBytes("T1"); 221 final byte[] T2 = Bytes.toBytes("T2"); 222 final byte[] T3 = Bytes.toBytes("T3"); 223 HColumnDescriptor hcd = 224 new HColumnDescriptor(FAMILY).setKeepDeletedCells(KeepDeletedCells.TRUE).setMaxVersions(3); 225 226 HTableDescriptor desc = new HTableDescriptor(tableName); 227 desc.addFamily(hcd); 228 TEST_UTIL.getAdmin().createTable(desc); 229 try (Table h = TEST_UTIL.getConnection().getTable(tableName)) { 230 long ts = System.currentTimeMillis(); 231 Put p = new Put(T1, ts); 232 p.addColumn(FAMILY, C0, T1); 233 h.put(p); 234 p = new Put(T1, ts + 2); 235 p.addColumn(FAMILY, C0, T2); 236 h.put(p); 237 p = new Put(T1, ts + 4); 238 p.addColumn(FAMILY, C0, T3); 239 h.put(p); 240 241 Delete d = new Delete(T1, ts + 3); 242 h.delete(d); 243 244 d = new Delete(T1, ts + 3); 245 d.addColumns(FAMILY, C0, ts + 3); 246 h.delete(d); 247 248 Get g = new Get(T1); 249 // does *not* include the delete 250 g.setTimeRange(0, ts + 3); 251 Result r = h.get(g); 252 assertArrayEquals(T2, r.getValue(FAMILY, C0)); 253 254 Scan s = new Scan(T1); 255 s.setTimeRange(0, ts + 3); 256 s.setMaxVersions(); 257 try (ResultScanner scanner = h.getScanner(s)) { 258 Cell[] kvs = scanner.next().rawCells(); 259 assertArrayEquals(T2, CellUtil.cloneValue(kvs[0])); 260 assertArrayEquals(T1, CellUtil.cloneValue(kvs[1])); 261 } 262 263 s = new Scan(T1); 264 s.setRaw(true); 265 s.setMaxVersions(); 266 try (ResultScanner scanner = h.getScanner(s)) { 267 Cell[] kvs = scanner.next().rawCells(); 268 assertTrue(PrivateCellUtil.isDeleteFamily(kvs[0])); 269 assertArrayEquals(T3, CellUtil.cloneValue(kvs[1])); 270 assertTrue(CellUtil.isDelete(kvs[2])); 271 assertArrayEquals(T2, CellUtil.cloneValue(kvs[3])); 272 assertArrayEquals(T1, CellUtil.cloneValue(kvs[4])); 273 } 274 } 275 } 276 277 /** 278 * Basic client side validation of HBASE-10118 279 */ 280 @Test 281 public void testPurgeFutureDeletes() throws Exception { 282 final TableName tableName = TableName.valueOf(name.getMethodName()); 283 final byte[] ROW = Bytes.toBytes("row"); 284 final byte[] FAMILY = Bytes.toBytes("family"); 285 final byte[] COLUMN = Bytes.toBytes("column"); 286 final byte[] VALUE = Bytes.toBytes("value"); 287 288 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) { 289 // future timestamp 290 long ts = System.currentTimeMillis() * 2; 291 Put put = new Put(ROW, ts); 292 put.addColumn(FAMILY, COLUMN, VALUE); 293 table.put(put); 294 295 Get get = new Get(ROW); 296 Result result = table.get(get); 297 assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN)); 298 299 Delete del = new Delete(ROW); 300 del.addColumn(FAMILY, COLUMN, ts); 301 table.delete(del); 302 303 get = new Get(ROW); 304 result = table.get(get); 305 assertNull(result.getValue(FAMILY, COLUMN)); 306 307 // major compaction, purged future deletes 308 TEST_UTIL.getAdmin().flush(tableName); 309 TEST_UTIL.getAdmin().majorCompact(tableName); 310 311 // waiting for the major compaction to complete 312 TEST_UTIL.waitFor(6000, new Waiter.Predicate<IOException>() { 313 @Override 314 public boolean evaluate() throws IOException { 315 return TEST_UTIL.getAdmin().getCompactionState(tableName) == CompactionState.NONE; 316 } 317 }); 318 319 put = new Put(ROW, ts); 320 put.addColumn(FAMILY, COLUMN, VALUE); 321 table.put(put); 322 323 get = new Get(ROW); 324 result = table.get(get); 325 assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN)); 326 } 327 } 328 329 /** 330 * Verifies that getConfiguration returns the same Configuration object used 331 * to create the HTable instance. 332 */ 333 @Test 334 public void testGetConfiguration() throws Exception { 335 final TableName tableName = TableName.valueOf(name.getMethodName()); 336 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") }; 337 Configuration conf = TEST_UTIL.getConfiguration(); 338 try (Table table = TEST_UTIL.createTable(tableName, FAMILIES)) { 339 assertSame(conf, table.getConfiguration()); 340 } 341 } 342 343 /** 344 * Test from client side of an involved filter against a multi family that 345 * involves deletes. 346 */ 347 @Test 348 public void testWeirdCacheBehaviour() throws Exception { 349 final TableName tableName = TableName.valueOf(name.getMethodName()); 350 byte [][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), 351 Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"), 352 Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") }; 353 try (Table ht = TEST_UTIL.createTable(tableName, FAMILIES)) { 354 String value = "this is the value"; 355 String value2 = "this is some other value"; 356 String keyPrefix1 = TEST_UTIL.getRandomUUID().toString(); 357 String keyPrefix2 = TEST_UTIL.getRandomUUID().toString(); 358 String keyPrefix3 = TEST_UTIL.getRandomUUID().toString(); 359 putRows(ht, 3, value, keyPrefix1); 360 putRows(ht, 3, value, keyPrefix2); 361 putRows(ht, 3, value, keyPrefix3); 362 putRows(ht, 3, value2, keyPrefix1); 363 putRows(ht, 3, value2, keyPrefix2); 364 putRows(ht, 3, value2, keyPrefix3); 365 try (Table table = TEST_UTIL.getConnection().getTable(tableName)) { 366 System.out.println("Checking values for key: " + keyPrefix1); 367 assertEquals("Got back incorrect number of rows from scan", 3, 368 getNumberOfRows(keyPrefix1, value2, table)); 369 System.out.println("Checking values for key: " + keyPrefix2); 370 assertEquals("Got back incorrect number of rows from scan", 3, 371 getNumberOfRows(keyPrefix2, value2, table)); 372 System.out.println("Checking values for key: " + keyPrefix3); 373 assertEquals("Got back incorrect number of rows from scan", 3, 374 getNumberOfRows(keyPrefix3, value2, table)); 375 deleteColumns(ht, value2, keyPrefix1); 376 deleteColumns(ht, value2, keyPrefix2); 377 deleteColumns(ht, value2, keyPrefix3); 378 System.out.println("Starting important checks....."); 379 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 380 0, getNumberOfRows(keyPrefix1, value2, table)); 381 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 382 0, getNumberOfRows(keyPrefix2, value2, table)); 383 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 384 0, getNumberOfRows(keyPrefix3, value2, table)); 385 } 386 } 387 } 388 389 private void deleteColumns(Table ht, String value, String keyPrefix) 390 throws IOException { 391 ResultScanner scanner = buildScanner(keyPrefix, value, ht); 392 Iterator<Result> it = scanner.iterator(); 393 int count = 0; 394 while (it.hasNext()) { 395 Result result = it.next(); 396 Delete delete = new Delete(result.getRow()); 397 delete.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2")); 398 ht.delete(delete); 399 count++; 400 } 401 assertEquals("Did not perform correct number of deletes", 3, count); 402 } 403 404 private int getNumberOfRows(String keyPrefix, String value, Table ht) 405 throws Exception { 406 ResultScanner resultScanner = buildScanner(keyPrefix, value, ht); 407 Iterator<Result> scanner = resultScanner.iterator(); 408 int numberOfResults = 0; 409 while (scanner.hasNext()) { 410 Result result = scanner.next(); 411 System.out.println("Got back key: " + Bytes.toString(result.getRow())); 412 for (Cell kv : result.rawCells()) { 413 System.out.println("kv=" + kv.toString() + ", " 414 + Bytes.toString(CellUtil.cloneValue(kv))); 415 } 416 numberOfResults++; 417 } 418 return numberOfResults; 419 } 420 421 private ResultScanner buildScanner(String keyPrefix, String value, Table ht) 422 throws IOException { 423 // OurFilterList allFilters = new OurFilterList(); 424 FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */); 425 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); 426 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes 427 .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOperator.EQUAL, Bytes 428 .toBytes(value)); 429 filter.setFilterIfMissing(true); 430 allFilters.addFilter(filter); 431 432 // allFilters.addFilter(new 433 // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"), 434 // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value))); 435 436 Scan scan = new Scan(); 437 scan.addFamily(Bytes.toBytes("trans-blob")); 438 scan.addFamily(Bytes.toBytes("trans-type")); 439 scan.addFamily(Bytes.toBytes("trans-date")); 440 scan.addFamily(Bytes.toBytes("trans-tags")); 441 scan.addFamily(Bytes.toBytes("trans-group")); 442 scan.setFilter(allFilters); 443 444 return ht.getScanner(scan); 445 } 446 447 private void putRows(Table ht, int numRows, String value, String key) 448 throws IOException { 449 for (int i = 0; i < numRows; i++) { 450 String row = key + "_" + TEST_UTIL.getRandomUUID().toString(); 451 System.out.println(String.format("Saving row: %s, with value %s", row, 452 value)); 453 Put put = new Put(Bytes.toBytes(row)); 454 put.setDurability(Durability.SKIP_WAL); 455 put.addColumn(Bytes.toBytes("trans-blob"), null, Bytes 456 .toBytes("value for blob")); 457 put.addColumn(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement")); 458 put.addColumn(Bytes.toBytes("trans-date"), null, Bytes 459 .toBytes("20090921010101999")); 460 put.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes 461 .toBytes(value)); 462 put.addColumn(Bytes.toBytes("trans-group"), null, Bytes 463 .toBytes("adhocTransactionGroupId")); 464 ht.put(put); 465 } 466 } 467 468 /** 469 * Test filters when multiple regions. It does counts. Needs eye-balling of 470 * logs to ensure that we're not scanning more regions that we're supposed to. 471 * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package. 472 */ 473 @Test 474 public void testFilterAcrossMultipleRegions() 475 throws IOException, InterruptedException { 476 final TableName tableName = TableName.valueOf(name.getMethodName()); 477 try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) { 478 int rowCount = TEST_UTIL.loadTable(t, FAMILY, false); 479 assertRowCount(t, rowCount); 480 // Split the table. Should split on a reasonable key; 'lqj' 481 List<HRegionLocation> regions = splitTable(t); 482 assertRowCount(t, rowCount); 483 // Get end key of first region. 484 byte[] endKey = regions.get(0).getRegionInfo().getEndKey(); 485 // Count rows with a filter that stops us before passed 'endKey'. 486 // Should be count of rows in first region. 487 int endKeyCount = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey)); 488 assertTrue(endKeyCount < rowCount); 489 490 // How do I know I did not got to second region? Thats tough. Can't really 491 // do that in client-side region test. I verified by tracing in debugger. 492 // I changed the messages that come out when set to DEBUG so should see 493 // when scanner is done. Says "Finished with scanning..." with region name. 494 // Check that its finished in right region. 495 496 // New test. Make it so scan goes into next region by one and then two. 497 // Make sure count comes out right. 498 byte[] key = new byte[]{endKey[0], endKey[1], (byte) (endKey[2] + 1)}; 499 int plusOneCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key)); 500 assertEquals(endKeyCount + 1, plusOneCount); 501 key = new byte[]{endKey[0], endKey[1], (byte) (endKey[2] + 2)}; 502 int plusTwoCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key)); 503 assertEquals(endKeyCount + 2, plusTwoCount); 504 505 // New test. Make it so I scan one less than endkey. 506 key = new byte[]{endKey[0], endKey[1], (byte) (endKey[2] - 1)}; 507 int minusOneCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key)); 508 assertEquals(endKeyCount - 1, minusOneCount); 509 // For above test... study logs. Make sure we do "Finished with scanning.." 510 // in first region and that we do not fall into the next region. 511 512 key = new byte[]{'a', 'a', 'a'}; 513 int countBBB = TEST_UTIL.countRows(t, 514 createScanWithRowFilter(key, null, CompareOperator.EQUAL)); 515 assertEquals(1, countBBB); 516 517 int countGreater = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey, null, 518 CompareOperator.GREATER_OR_EQUAL)); 519 // Because started at start of table. 520 assertEquals(0, countGreater); 521 countGreater = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey, endKey, 522 CompareOperator.GREATER_OR_EQUAL)); 523 assertEquals(rowCount - endKeyCount, countGreater); 524 } 525 } 526 527 /* 528 * @param key 529 * @return Scan with RowFilter that does LESS than passed key. 530 */ 531 private Scan createScanWithRowFilter(final byte [] key) { 532 return createScanWithRowFilter(key, null, CompareOperator.LESS); 533 } 534 535 /* 536 * @param key 537 * @param op 538 * @param startRow 539 * @return Scan with RowFilter that does CompareOp op on passed key. 540 */ 541 private Scan createScanWithRowFilter(final byte [] key, 542 final byte [] startRow, CompareOperator op) { 543 // Make sure key is of some substance... non-null and > than first key. 544 assertTrue(key != null && key.length > 0 && 545 Bytes.BYTES_COMPARATOR.compare(key, new byte [] {'a', 'a', 'a'}) >= 0); 546 LOG.info("Key=" + Bytes.toString(key)); 547 Scan s = startRow == null? new Scan(): new Scan(startRow); 548 Filter f = new RowFilter(op, new BinaryComparator(key)); 549 f = new WhileMatchFilter(f); 550 s.setFilter(f); 551 return s; 552 } 553 554 private void assertRowCount(final Table t, final int expected) 555 throws IOException { 556 assertEquals(expected, TEST_UTIL.countRows(t, new Scan())); 557 } 558 559 /* 560 * Split table into multiple regions. 561 * @param t Table to split. 562 * @return Map of regions to servers. 563 * @throws IOException 564 */ 565 private List<HRegionLocation> splitTable(final Table t) 566 throws IOException, InterruptedException { 567 // Split this table in two. 568 try (Admin admin = TEST_UTIL.getAdmin()) { 569 admin.split(t.getName()); 570 } 571 List<HRegionLocation> regions = waitOnSplit(t); 572 assertTrue(regions.size() > 1); 573 return regions; 574 } 575 576 /* 577 * Wait on table split. May return because we waited long enough on the split 578 * and it didn't happen. Caller should check. 579 * @param t 580 * @return Map of table regions; caller needs to check table actually split. 581 */ 582 private List<HRegionLocation> waitOnSplit(final Table t) 583 throws IOException { 584 try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(t.getName())) { 585 List<HRegionLocation> regions = locator.getAllRegionLocations(); 586 int originalCount = regions.size(); 587 for (int i = 0; i < TEST_UTIL.getConfiguration().getInt("hbase.test.retries", 30); i++) { 588 Thread.currentThread(); 589 try { 590 Thread.sleep(1000); 591 } catch (InterruptedException e) { 592 e.printStackTrace(); 593 } 594 regions = locator.getAllRegionLocations(); 595 if (regions.size() > originalCount) 596 break; 597 } 598 return regions; 599 } 600 } 601 602 @Test 603 public void testSuperSimple() throws Exception { 604 final TableName tableName = TableName.valueOf(name.getMethodName()); 605 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 606 Put put = new Put(ROW); 607 put.addColumn(FAMILY, QUALIFIER, VALUE); 608 ht.put(put); 609 Scan scan = new Scan(); 610 scan.addColumn(FAMILY, tableName.toBytes()); 611 try (ResultScanner scanner = ht.getScanner(scan)) { 612 Result result = scanner.next(); 613 assertTrue("Expected null result", result == null); 614 } 615 } 616 } 617 618 @Test 619 public void testMaxKeyValueSize() throws Exception { 620 final TableName tableName = TableName.valueOf(name.getMethodName()); 621 Configuration conf = TEST_UTIL.getConfiguration(); 622 String oldMaxSize = conf.get(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY); 623 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 624 byte[] value = new byte[4 * 1024 * 1024]; 625 Put put = new Put(ROW); 626 put.addColumn(FAMILY, QUALIFIER, value); 627 ht.put(put); 628 629 try { 630 TEST_UTIL.getConfiguration().setInt( 631 ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, 2 * 1024 * 1024); 632 // Create new table so we pick up the change in Configuration. 633 try (Connection connection = 634 ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) { 635 try (Table t = connection.getTable(TableName.valueOf(FAMILY))) { 636 put = new Put(ROW); 637 put.addColumn(FAMILY, QUALIFIER, value); 638 t.put(put); 639 } 640 } 641 fail("Inserting a too large KeyValue worked, should throw exception"); 642 } catch (Exception e) { 643 } 644 } 645 conf.set(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, oldMaxSize); 646 } 647 648 @Test 649 public void testFilters() throws Exception { 650 final TableName tableName = TableName.valueOf(name.getMethodName()); 651 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 652 byte[][] ROWS = makeN(ROW, 10); 653 byte[][] QUALIFIERS = { 654 Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"), 655 Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"), 656 Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"), 657 Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"), 658 Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>") 659 }; 660 for (int i = 0; i < 10; i++) { 661 Put put = new Put(ROWS[i]); 662 put.setDurability(Durability.SKIP_WAL); 663 put.addColumn(FAMILY, QUALIFIERS[i], VALUE); 664 ht.put(put); 665 } 666 Scan scan = new Scan(); 667 scan.addFamily(FAMILY); 668 Filter filter = new QualifierFilter(CompareOperator.EQUAL, 669 new RegexStringComparator("col[1-5]")); 670 scan.setFilter(filter); 671 try (ResultScanner scanner = ht.getScanner(scan)) { 672 int expectedIndex = 1; 673 for (Result result : scanner) { 674 assertEquals(1, result.size()); 675 assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[expectedIndex])); 676 assertTrue(Bytes.equals(CellUtil.cloneQualifier(result.rawCells()[0]), 677 QUALIFIERS[expectedIndex])); 678 expectedIndex++; 679 } 680 assertEquals(6, expectedIndex); 681 } 682 } 683 } 684 685 @Test 686 public void testFilterWithLongCompartor() throws Exception { 687 final TableName tableName = TableName.valueOf(name.getMethodName()); 688 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 689 byte[][] ROWS = makeN(ROW, 10); 690 byte[][] values = new byte[10][]; 691 for (int i = 0; i < 10; i++) { 692 values[i] = Bytes.toBytes(100L * i); 693 } 694 for (int i = 0; i < 10; i++) { 695 Put put = new Put(ROWS[i]); 696 put.setDurability(Durability.SKIP_WAL); 697 put.addColumn(FAMILY, QUALIFIER, values[i]); 698 ht.put(put); 699 } 700 Scan scan = new Scan(); 701 scan.addFamily(FAMILY); 702 Filter filter = new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOperator.GREATER, 703 new LongComparator(500)); 704 scan.setFilter(filter); 705 try (ResultScanner scanner = ht.getScanner(scan)) { 706 int expectedIndex = 0; 707 for (Result result : scanner) { 708 assertEquals(1, result.size()); 709 assertTrue(Bytes.toLong(result.getValue(FAMILY, QUALIFIER)) > 500); 710 expectedIndex++; 711 } 712 assertEquals(4, expectedIndex); 713 } 714 } 715 } 716 717 @Test 718 public void testKeyOnlyFilter() throws Exception { 719 final TableName tableName = TableName.valueOf(name.getMethodName()); 720 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 721 byte[][] ROWS = makeN(ROW, 10); 722 byte[][] QUALIFIERS = { 723 Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"), 724 Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"), 725 Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"), 726 Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"), 727 Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>") 728 }; 729 for (int i = 0; i < 10; i++) { 730 Put put = new Put(ROWS[i]); 731 put.setDurability(Durability.SKIP_WAL); 732 put.addColumn(FAMILY, QUALIFIERS[i], VALUE); 733 ht.put(put); 734 } 735 Scan scan = new Scan(); 736 scan.addFamily(FAMILY); 737 Filter filter = new KeyOnlyFilter(true); 738 scan.setFilter(filter); 739 try (ResultScanner scanner = ht.getScanner(scan)) { 740 int count = 0; 741 for (Result result : scanner) { 742 assertEquals(1, result.size()); 743 assertEquals(Bytes.SIZEOF_INT, result.rawCells()[0].getValueLength()); 744 assertEquals(VALUE.length, Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0]))); 745 count++; 746 } 747 assertEquals(10, count); 748 } 749 } 750 } 751 752 /** 753 * Test simple table and non-existent row cases. 754 */ 755 @Test 756 public void testSimpleMissing() throws Exception { 757 final TableName tableName = TableName.valueOf(name.getMethodName()); 758 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 759 byte[][] ROWS = makeN(ROW, 4); 760 761 // Try to get a row on an empty table 762 Get get = new Get(ROWS[0]); 763 Result result = ht.get(get); 764 assertEmptyResult(result); 765 766 get = new Get(ROWS[0]); 767 get.addFamily(FAMILY); 768 result = ht.get(get); 769 assertEmptyResult(result); 770 771 get = new Get(ROWS[0]); 772 get.addColumn(FAMILY, QUALIFIER); 773 result = ht.get(get); 774 assertEmptyResult(result); 775 776 Scan scan = new Scan(); 777 result = getSingleScanResult(ht, scan); 778 assertNullResult(result); 779 780 scan = new Scan(ROWS[0]); 781 result = getSingleScanResult(ht, scan); 782 assertNullResult(result); 783 784 scan = new Scan(ROWS[0], ROWS[1]); 785 result = getSingleScanResult(ht, scan); 786 assertNullResult(result); 787 788 scan = new Scan(); 789 scan.addFamily(FAMILY); 790 result = getSingleScanResult(ht, scan); 791 assertNullResult(result); 792 793 scan = new Scan(); 794 scan.addColumn(FAMILY, QUALIFIER); 795 result = getSingleScanResult(ht, scan); 796 assertNullResult(result); 797 798 // Insert a row 799 800 Put put = new Put(ROWS[2]); 801 put.addColumn(FAMILY, QUALIFIER, VALUE); 802 ht.put(put); 803 804 // Try to get empty rows around it 805 806 get = new Get(ROWS[1]); 807 result = ht.get(get); 808 assertEmptyResult(result); 809 810 get = new Get(ROWS[0]); 811 get.addFamily(FAMILY); 812 result = ht.get(get); 813 assertEmptyResult(result); 814 815 get = new Get(ROWS[3]); 816 get.addColumn(FAMILY, QUALIFIER); 817 result = ht.get(get); 818 assertEmptyResult(result); 819 820 // Try to scan empty rows around it 821 822 scan = new Scan(ROWS[3]); 823 result = getSingleScanResult(ht, scan); 824 assertNullResult(result); 825 826 scan = new Scan(ROWS[0], ROWS[2]); 827 result = getSingleScanResult(ht, scan); 828 assertNullResult(result); 829 830 // Make sure we can actually get the row 831 832 get = new Get(ROWS[2]); 833 result = ht.get(get); 834 assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE); 835 836 get = new Get(ROWS[2]); 837 get.addFamily(FAMILY); 838 result = ht.get(get); 839 assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE); 840 841 get = new Get(ROWS[2]); 842 get.addColumn(FAMILY, QUALIFIER); 843 result = ht.get(get); 844 assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE); 845 846 // Make sure we can scan the row 847 848 scan = new Scan(); 849 result = getSingleScanResult(ht, scan); 850 assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE); 851 852 scan = new Scan(ROWS[0], ROWS[3]); 853 result = getSingleScanResult(ht, scan); 854 assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE); 855 856 scan = new Scan(ROWS[2], ROWS[3]); 857 result = getSingleScanResult(ht, scan); 858 assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE); 859 } 860 } 861 862 /** 863 * Test basic puts, gets, scans, and deletes for a single row 864 * in a multiple family table. 865 */ 866 @Test 867 public void testSingleRowMultipleFamily() throws Exception { 868 final TableName tableName = TableName.valueOf(name.getMethodName()); 869 byte [][] ROWS = makeN(ROW, 3); 870 byte [][] FAMILIES = makeNAscii(FAMILY, 10); 871 byte [][] QUALIFIERS = makeN(QUALIFIER, 10); 872 byte [][] VALUES = makeN(VALUE, 10); 873 874 try (Table ht = TEST_UTIL.createTable(tableName, FAMILIES)) { 875 876 Get get; 877 Scan scan; 878 Delete delete; 879 Put put; 880 Result result; 881 882 //////////////////////////////////////////////////////////////////////////// 883 // Insert one column to one family 884 //////////////////////////////////////////////////////////////////////////// 885 886 put = new Put(ROWS[0]); 887 put.addColumn(FAMILIES[4], QUALIFIERS[0], VALUES[0]); 888 ht.put(put); 889 890 // Get the single column 891 getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0); 892 893 // Scan the single column 894 scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0); 895 896 // Get empty results around inserted column 897 getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0); 898 899 // Scan empty results around inserted column 900 scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0); 901 902 //////////////////////////////////////////////////////////////////////////// 903 // Flush memstore and run same tests from storefiles 904 //////////////////////////////////////////////////////////////////////////// 905 906 TEST_UTIL.flush(); 907 908 // Redo get and scan tests from storefile 909 getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0); 910 scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0); 911 getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0); 912 scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0); 913 914 //////////////////////////////////////////////////////////////////////////// 915 // Now, Test reading from memstore and storefiles at once 916 //////////////////////////////////////////////////////////////////////////// 917 918 // Insert multiple columns to two other families 919 put = new Put(ROWS[0]); 920 put.addColumn(FAMILIES[2], QUALIFIERS[2], VALUES[2]); 921 put.addColumn(FAMILIES[2], QUALIFIERS[4], VALUES[4]); 922 put.addColumn(FAMILIES[4], QUALIFIERS[4], VALUES[4]); 923 put.addColumn(FAMILIES[6], QUALIFIERS[6], VALUES[6]); 924 put.addColumn(FAMILIES[6], QUALIFIERS[7], VALUES[7]); 925 put.addColumn(FAMILIES[7], QUALIFIERS[7], VALUES[7]); 926 put.addColumn(FAMILIES[9], QUALIFIERS[0], VALUES[0]); 927 ht.put(put); 928 929 // Get multiple columns across multiple families and get empties around it 930 singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES); 931 932 // Scan multiple columns across multiple families and scan empties around it 933 singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES); 934 935 //////////////////////////////////////////////////////////////////////////// 936 // Flush the table again 937 //////////////////////////////////////////////////////////////////////////// 938 939 TEST_UTIL.flush(); 940 941 // Redo tests again 942 singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES); 943 singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES); 944 945 // Insert more data to memstore 946 put = new Put(ROWS[0]); 947 put.addColumn(FAMILIES[6], QUALIFIERS[5], VALUES[5]); 948 put.addColumn(FAMILIES[6], QUALIFIERS[8], VALUES[8]); 949 put.addColumn(FAMILIES[6], QUALIFIERS[9], VALUES[9]); 950 put.addColumn(FAMILIES[4], QUALIFIERS[3], VALUES[3]); 951 ht.put(put); 952 953 //////////////////////////////////////////////////////////////////////////// 954 // Delete a storefile column 955 //////////////////////////////////////////////////////////////////////////// 956 delete = new Delete(ROWS[0]); 957 delete.addColumns(FAMILIES[6], QUALIFIERS[7]); 958 ht.delete(delete); 959 960 // Try to get deleted column 961 get = new Get(ROWS[0]); 962 get.addColumn(FAMILIES[6], QUALIFIERS[7]); 963 result = ht.get(get); 964 assertEmptyResult(result); 965 966 // Try to scan deleted column 967 scan = new Scan(); 968 scan.addColumn(FAMILIES[6], QUALIFIERS[7]); 969 result = getSingleScanResult(ht, scan); 970 assertNullResult(result); 971 972 // Make sure we can still get a column before it and after it 973 get = new Get(ROWS[0]); 974 get.addColumn(FAMILIES[6], QUALIFIERS[6]); 975 result = ht.get(get); 976 assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]); 977 978 get = new Get(ROWS[0]); 979 get.addColumn(FAMILIES[6], QUALIFIERS[8]); 980 result = ht.get(get); 981 assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]); 982 983 // Make sure we can still scan a column before it and after it 984 scan = new Scan(); 985 scan.addColumn(FAMILIES[6], QUALIFIERS[6]); 986 result = getSingleScanResult(ht, scan); 987 assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]); 988 989 scan = new Scan(); 990 scan.addColumn(FAMILIES[6], QUALIFIERS[8]); 991 result = getSingleScanResult(ht, scan); 992 assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]); 993 994 //////////////////////////////////////////////////////////////////////////// 995 // Delete a memstore column 996 //////////////////////////////////////////////////////////////////////////// 997 delete = new Delete(ROWS[0]); 998 delete.addColumns(FAMILIES[6], QUALIFIERS[8]); 999 ht.delete(delete); 1000 1001 // Try to get deleted column 1002 get = new Get(ROWS[0]); 1003 get.addColumn(FAMILIES[6], QUALIFIERS[8]); 1004 result = ht.get(get); 1005 assertEmptyResult(result); 1006 1007 // Try to scan deleted column 1008 scan = new Scan(); 1009 scan.addColumn(FAMILIES[6], QUALIFIERS[8]); 1010 result = getSingleScanResult(ht, scan); 1011 assertNullResult(result); 1012 1013 // Make sure we can still get a column before it and after it 1014 get = new Get(ROWS[0]); 1015 get.addColumn(FAMILIES[6], QUALIFIERS[6]); 1016 result = ht.get(get); 1017 assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]); 1018 1019 get = new Get(ROWS[0]); 1020 get.addColumn(FAMILIES[6], QUALIFIERS[9]); 1021 result = ht.get(get); 1022 assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]); 1023 1024 // Make sure we can still scan a column before it and after it 1025 scan = new Scan(); 1026 scan.addColumn(FAMILIES[6], QUALIFIERS[6]); 1027 result = getSingleScanResult(ht, scan); 1028 assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]); 1029 1030 scan = new Scan(); 1031 scan.addColumn(FAMILIES[6], QUALIFIERS[9]); 1032 result = getSingleScanResult(ht, scan); 1033 assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]); 1034 1035 //////////////////////////////////////////////////////////////////////////// 1036 // Delete joint storefile/memstore family 1037 //////////////////////////////////////////////////////////////////////////// 1038 1039 delete = new Delete(ROWS[0]); 1040 delete.addFamily(FAMILIES[4]); 1041 ht.delete(delete); 1042 1043 // Try to get storefile column in deleted family 1044 get = new Get(ROWS[0]); 1045 get.addColumn(FAMILIES[4], QUALIFIERS[4]); 1046 result = ht.get(get); 1047 assertEmptyResult(result); 1048 1049 // Try to get memstore column in deleted family 1050 get = new Get(ROWS[0]); 1051 get.addColumn(FAMILIES[4], QUALIFIERS[3]); 1052 result = ht.get(get); 1053 assertEmptyResult(result); 1054 1055 // Try to get deleted family 1056 get = new Get(ROWS[0]); 1057 get.addFamily(FAMILIES[4]); 1058 result = ht.get(get); 1059 assertEmptyResult(result); 1060 1061 // Try to scan storefile column in deleted family 1062 scan = new Scan(); 1063 scan.addColumn(FAMILIES[4], QUALIFIERS[4]); 1064 result = getSingleScanResult(ht, scan); 1065 assertNullResult(result); 1066 1067 // Try to scan memstore column in deleted family 1068 scan = new Scan(); 1069 scan.addColumn(FAMILIES[4], QUALIFIERS[3]); 1070 result = getSingleScanResult(ht, scan); 1071 assertNullResult(result); 1072 1073 // Try to scan deleted family 1074 scan = new Scan(); 1075 scan.addFamily(FAMILIES[4]); 1076 result = getSingleScanResult(ht, scan); 1077 assertNullResult(result); 1078 1079 // Make sure we can still get another family 1080 get = new Get(ROWS[0]); 1081 get.addColumn(FAMILIES[2], QUALIFIERS[2]); 1082 result = ht.get(get); 1083 assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]); 1084 1085 get = new Get(ROWS[0]); 1086 get.addColumn(FAMILIES[6], QUALIFIERS[9]); 1087 result = ht.get(get); 1088 assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]); 1089 1090 // Make sure we can still scan another family 1091 scan = new Scan(); 1092 scan.addColumn(FAMILIES[6], QUALIFIERS[6]); 1093 result = getSingleScanResult(ht, scan); 1094 assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]); 1095 1096 scan = new Scan(); 1097 scan.addColumn(FAMILIES[6], QUALIFIERS[9]); 1098 result = getSingleScanResult(ht, scan); 1099 assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]); 1100 1101 //////////////////////////////////////////////////////////////////////////// 1102 // Flush everything and rerun delete tests 1103 //////////////////////////////////////////////////////////////////////////// 1104 1105 TEST_UTIL.flush(); 1106 1107 // Try to get storefile column in deleted family 1108 get = new Get(ROWS[0]); 1109 get.addColumn(FAMILIES[4], QUALIFIERS[4]); 1110 result = ht.get(get); 1111 assertEmptyResult(result); 1112 1113 // Try to get memstore column in deleted family 1114 get = new Get(ROWS[0]); 1115 get.addColumn(FAMILIES[4], QUALIFIERS[3]); 1116 result = ht.get(get); 1117 assertEmptyResult(result); 1118 1119 // Try to get deleted family 1120 get = new Get(ROWS[0]); 1121 get.addFamily(FAMILIES[4]); 1122 result = ht.get(get); 1123 assertEmptyResult(result); 1124 1125 // Try to scan storefile column in deleted family 1126 scan = new Scan(); 1127 scan.addColumn(FAMILIES[4], QUALIFIERS[4]); 1128 result = getSingleScanResult(ht, scan); 1129 assertNullResult(result); 1130 1131 // Try to scan memstore column in deleted family 1132 scan = new Scan(); 1133 scan.addColumn(FAMILIES[4], QUALIFIERS[3]); 1134 result = getSingleScanResult(ht, scan); 1135 assertNullResult(result); 1136 1137 // Try to scan deleted family 1138 scan = new Scan(); 1139 scan.addFamily(FAMILIES[4]); 1140 result = getSingleScanResult(ht, scan); 1141 assertNullResult(result); 1142 1143 // Make sure we can still get another family 1144 get = new Get(ROWS[0]); 1145 get.addColumn(FAMILIES[2], QUALIFIERS[2]); 1146 result = ht.get(get); 1147 assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]); 1148 1149 get = new Get(ROWS[0]); 1150 get.addColumn(FAMILIES[6], QUALIFIERS[9]); 1151 result = ht.get(get); 1152 assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]); 1153 1154 // Make sure we can still scan another family 1155 scan = new Scan(); 1156 scan.addColumn(FAMILIES[6], QUALIFIERS[6]); 1157 result = getSingleScanResult(ht, scan); 1158 assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]); 1159 1160 scan = new Scan(); 1161 scan.addColumn(FAMILIES[6], QUALIFIERS[9]); 1162 result = getSingleScanResult(ht, scan); 1163 assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]); 1164 } 1165 } 1166 1167 @Test(expected = NullPointerException.class) 1168 public void testNullTableName() throws IOException { 1169 // Null table name (should NOT work) 1170 TEST_UTIL.createTable((TableName)null, FAMILY); 1171 fail("Creating a table with null name passed, should have failed"); 1172 } 1173 1174 @Test(expected = IllegalArgumentException.class) 1175 public void testNullFamilyName() throws IOException { 1176 final TableName tableName = TableName.valueOf(name.getMethodName()); 1177 1178 // Null family (should NOT work) 1179 TEST_UTIL.createTable(tableName, new byte[][]{null}); 1180 fail("Creating a table with a null family passed, should fail"); 1181 } 1182 1183 @Test 1184 public void testNullRowAndQualifier() throws Exception { 1185 final TableName tableName = TableName.valueOf(name.getMethodName()); 1186 1187 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 1188 1189 // Null row (should NOT work) 1190 try { 1191 Put put = new Put((byte[]) null); 1192 put.addColumn(FAMILY, QUALIFIER, VALUE); 1193 ht.put(put); 1194 fail("Inserting a null row worked, should throw exception"); 1195 } catch (Exception e) { 1196 } 1197 1198 // Null qualifier (should work) 1199 { 1200 Put put = new Put(ROW); 1201 put.addColumn(FAMILY, null, VALUE); 1202 ht.put(put); 1203 1204 getTestNull(ht, ROW, FAMILY, VALUE); 1205 1206 scanTestNull(ht, ROW, FAMILY, VALUE); 1207 1208 Delete delete = new Delete(ROW); 1209 delete.addColumns(FAMILY, null); 1210 ht.delete(delete); 1211 1212 Get get = new Get(ROW); 1213 Result result = ht.get(get); 1214 assertEmptyResult(result); 1215 } 1216 } 1217 } 1218 1219 @Test 1220 public void testNullEmptyQualifier() throws Exception { 1221 final TableName tableName = TableName.valueOf(name.getMethodName()); 1222 1223 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 1224 1225 // Empty qualifier, byte[0] instead of null (should work) 1226 try { 1227 Put put = new Put(ROW); 1228 put.addColumn(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE); 1229 ht.put(put); 1230 1231 getTestNull(ht, ROW, FAMILY, VALUE); 1232 1233 scanTestNull(ht, ROW, FAMILY, VALUE); 1234 1235 // Flush and try again 1236 1237 TEST_UTIL.flush(); 1238 1239 getTestNull(ht, ROW, FAMILY, VALUE); 1240 1241 scanTestNull(ht, ROW, FAMILY, VALUE); 1242 1243 Delete delete = new Delete(ROW); 1244 delete.addColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY); 1245 ht.delete(delete); 1246 1247 Get get = new Get(ROW); 1248 Result result = ht.get(get); 1249 assertEmptyResult(result); 1250 1251 } catch (Exception e) { 1252 throw new IOException("Using a row with null qualifier should not throw exception"); 1253 } 1254 } 1255 } 1256 1257 @Test 1258 public void testNullValue() throws IOException { 1259 final TableName tableName = TableName.valueOf(name.getMethodName()); 1260 1261 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 1262 // Null value 1263 try { 1264 Put put = new Put(ROW); 1265 put.addColumn(FAMILY, QUALIFIER, null); 1266 ht.put(put); 1267 1268 Get get = new Get(ROW); 1269 get.addColumn(FAMILY, QUALIFIER); 1270 Result result = ht.get(get); 1271 assertSingleResult(result, ROW, FAMILY, QUALIFIER, null); 1272 1273 Scan scan = new Scan(); 1274 scan.addColumn(FAMILY, QUALIFIER); 1275 result = getSingleScanResult(ht, scan); 1276 assertSingleResult(result, ROW, FAMILY, QUALIFIER, null); 1277 1278 Delete delete = new Delete(ROW); 1279 delete.addColumns(FAMILY, QUALIFIER); 1280 ht.delete(delete); 1281 1282 get = new Get(ROW); 1283 result = ht.get(get); 1284 assertEmptyResult(result); 1285 1286 } catch (Exception e) { 1287 throw new IOException("Null values should be allowed, but threw exception"); 1288 } 1289 } 1290 } 1291 1292 @Test 1293 public void testNullQualifier() throws Exception { 1294 final TableName tableName = TableName.valueOf(name.getMethodName()); 1295 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) { 1296 1297 // Work for Put 1298 Put put = new Put(ROW); 1299 put.addColumn(FAMILY, null, VALUE); 1300 table.put(put); 1301 1302 // Work for Get, Scan 1303 getTestNull(table, ROW, FAMILY, VALUE); 1304 scanTestNull(table, ROW, FAMILY, VALUE); 1305 1306 // Work for Delete 1307 Delete delete = new Delete(ROW); 1308 delete.addColumns(FAMILY, null); 1309 table.delete(delete); 1310 1311 Get get = new Get(ROW); 1312 Result result = table.get(get); 1313 assertEmptyResult(result); 1314 1315 // Work for Increment/Append 1316 Increment increment = new Increment(ROW); 1317 increment.addColumn(FAMILY, null, 1L); 1318 table.increment(increment); 1319 getTestNull(table, ROW, FAMILY, 1L); 1320 1321 table.incrementColumnValue(ROW, FAMILY, null, 1L); 1322 getTestNull(table, ROW, FAMILY, 2L); 1323 1324 delete = new Delete(ROW); 1325 delete.addColumns(FAMILY, null); 1326 table.delete(delete); 1327 1328 Append append = new Append(ROW); 1329 append.addColumn(FAMILY, null, VALUE); 1330 table.append(append); 1331 getTestNull(table, ROW, FAMILY, VALUE); 1332 1333 // Work for checkAndMutate using thenPut, thenMutate and thenDelete 1334 put = new Put(ROW); 1335 put.addColumn(FAMILY, null, Bytes.toBytes("checkAndPut")); 1336 table.put(put); 1337 table.checkAndMutate(ROW, FAMILY).ifEquals(VALUE).thenPut(put); 1338 1339 RowMutations mutate = new RowMutations(ROW); 1340 mutate.add(new Put(ROW).addColumn(FAMILY, null, Bytes.toBytes("checkAndMutate"))); 1341 table.checkAndMutate(ROW, FAMILY).ifEquals(Bytes.toBytes("checkAndPut")).thenMutate(mutate); 1342 1343 delete = new Delete(ROW); 1344 delete.addColumns(FAMILY, null); 1345 table.checkAndMutate(ROW, FAMILY).ifEquals(Bytes.toBytes("checkAndMutate")) 1346 .thenDelete(delete); 1347 } 1348 } 1349 1350 @Test 1351 public void testVersions() throws Exception { 1352 final TableName tableName = TableName.valueOf(name.getMethodName()); 1353 1354 long [] STAMPS = makeStamps(20); 1355 byte [][] VALUES = makeNAscii(VALUE, 20); 1356 1357 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) { 1358 1359 // Insert 4 versions of same column 1360 Put put = new Put(ROW); 1361 put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 1362 put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 1363 put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 1364 put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 1365 ht.put(put); 1366 1367 // Verify we can get each one properly 1368 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 1369 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 1370 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 1371 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 1372 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 1373 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 1374 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 1375 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 1376 1377 // Verify we don't accidentally get others 1378 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 1379 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); 1380 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); 1381 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 1382 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); 1383 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); 1384 1385 // Ensure maxVersions in query is respected 1386 Get get = new Get(ROW); 1387 get.addColumn(FAMILY, QUALIFIER); 1388 get.setMaxVersions(2); 1389 Result result = ht.get(get); 1390 assertNResult(result, ROW, FAMILY, QUALIFIER, 1391 new long [] {STAMPS[4], STAMPS[5]}, 1392 new byte[][] {VALUES[4], VALUES[5]}, 1393 0, 1); 1394 1395 Scan scan = new Scan(ROW); 1396 scan.addColumn(FAMILY, QUALIFIER); 1397 scan.setMaxVersions(2); 1398 result = getSingleScanResult(ht, scan); 1399 assertNResult(result, ROW, FAMILY, QUALIFIER, 1400 new long[]{STAMPS[4], STAMPS[5]}, 1401 new byte[][]{VALUES[4], VALUES[5]}, 1402 0, 1); 1403 1404 // Flush and redo 1405 1406 TEST_UTIL.flush(); 1407 1408 // Verify we can get each one properly 1409 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 1410 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 1411 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 1412 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 1413 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 1414 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 1415 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 1416 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 1417 1418 // Verify we don't accidentally get others 1419 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 1420 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); 1421 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); 1422 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 1423 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); 1424 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); 1425 1426 // Ensure maxVersions in query is respected 1427 get = new Get(ROW); 1428 get.addColumn(FAMILY, QUALIFIER); 1429 get.setMaxVersions(2); 1430 result = ht.get(get); 1431 assertNResult(result, ROW, FAMILY, QUALIFIER, 1432 new long [] {STAMPS[4], STAMPS[5]}, 1433 new byte[][] {VALUES[4], VALUES[5]}, 1434 0, 1); 1435 1436 scan = new Scan(ROW); 1437 scan.addColumn(FAMILY, QUALIFIER); 1438 scan.setMaxVersions(2); 1439 result = getSingleScanResult(ht, scan); 1440 assertNResult(result, ROW, FAMILY, QUALIFIER, 1441 new long[]{STAMPS[4], STAMPS[5]}, 1442 new byte[][]{VALUES[4], VALUES[5]}, 1443 0, 1); 1444 1445 1446 // Add some memstore and retest 1447 1448 // Insert 4 more versions of same column and a dupe 1449 put = new Put(ROW); 1450 put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); 1451 put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]); 1452 put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]); 1453 put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]); 1454 ht.put(put); 1455 1456 // Ensure maxVersions in query is respected 1457 get = new Get(ROW); 1458 get.addColumn(FAMILY, QUALIFIER); 1459 get.setMaxVersions(); 1460 result = ht.get(get); 1461 assertNResult(result, ROW, FAMILY, QUALIFIER, 1462 new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], 1463 STAMPS[8]}, 1464 new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], 1465 VALUES[8]}, 1466 0, 7); 1467 1468 scan = new Scan(ROW); 1469 scan.addColumn(FAMILY, QUALIFIER); 1470 scan.setMaxVersions(); 1471 result = getSingleScanResult(ht, scan); 1472 assertNResult(result, ROW, FAMILY, QUALIFIER, 1473 new long[]{STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], 1474 STAMPS[8]}, 1475 new byte[][]{VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], 1476 VALUES[8]},0, 7); 1477 1478 get = new Get(ROW); 1479 get.setMaxVersions(); 1480 result = ht.get(get); 1481 assertNResult(result, ROW, FAMILY, QUALIFIER, 1482 new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], 1483 STAMPS[8]}, 1484 new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], 1485 VALUES[8]}, 1486 0, 7); 1487 1488 scan = new Scan(ROW); 1489 scan.setMaxVersions(); 1490 result = getSingleScanResult(ht, scan); 1491 assertNResult(result, ROW, FAMILY, QUALIFIER, 1492 new long[]{STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], 1493 STAMPS[8]}, 1494 new byte[][]{VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], 1495 VALUES[8]},0, 7); 1496 1497 // Verify we can get each one properly 1498 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 1499 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 1500 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 1501 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]); 1502 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 1503 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 1504 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 1505 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]); 1506 1507 // Verify we don't accidentally get others 1508 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 1509 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]); 1510 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 1511 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]); 1512 1513 // Ensure maxVersions of table is respected 1514 1515 TEST_UTIL.flush(); 1516 1517 // Insert 4 more versions of same column and a dupe 1518 put = new Put(ROW); 1519 put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]); 1520 put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]); 1521 put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]); 1522 put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]); 1523 ht.put(put); 1524 1525 get = new Get(ROW); 1526 get.addColumn(FAMILY, QUALIFIER); 1527 get.setMaxVersions(Integer.MAX_VALUE); 1528 result = ht.get(get); 1529 assertNResult(result, ROW, FAMILY, QUALIFIER, 1530 new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], 1531 STAMPS[11], STAMPS[13], STAMPS[15]}, 1532 new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], 1533 VALUES[11], VALUES[13], VALUES[15]}, 1534 0, 9); 1535 1536 scan = new Scan(ROW); 1537 scan.addColumn(FAMILY, QUALIFIER); 1538 scan.setMaxVersions(Integer.MAX_VALUE); 1539 result = getSingleScanResult(ht, scan); 1540 assertNResult(result, ROW, FAMILY, QUALIFIER, 1541 new long[]{STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], 1542 STAMPS[11], STAMPS[13], STAMPS[15]}, 1543 new byte[][]{VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], 1544 VALUES[11], VALUES[13], VALUES[15]},0, 9); 1545 1546 // Delete a version in the memstore and a version in a storefile 1547 Delete delete = new Delete(ROW); 1548 delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]); 1549 delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]); 1550 ht.delete(delete); 1551 1552 // Test that it's gone 1553 get = new Get(ROW); 1554 get.addColumn(FAMILY, QUALIFIER); 1555 get.setMaxVersions(Integer.MAX_VALUE); 1556 result = ht.get(get); 1557 assertNResult(result, ROW, FAMILY, QUALIFIER, 1558 new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], 1559 STAMPS[9], STAMPS[13], STAMPS[15]}, 1560 new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], 1561 VALUES[9], VALUES[13], VALUES[15]}, 1562 0, 9); 1563 1564 scan = new Scan(ROW); 1565 scan.addColumn(FAMILY, QUALIFIER); 1566 scan.setMaxVersions(Integer.MAX_VALUE); 1567 result = getSingleScanResult(ht, scan); 1568 assertNResult(result, ROW, FAMILY, QUALIFIER, 1569 new long[]{STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], 1570 STAMPS[9], STAMPS[13], STAMPS[15]}, 1571 new byte[][]{VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], 1572 VALUES[9], VALUES[13], VALUES[15]},0, 9); 1573 } 1574 } 1575 1576 @Test 1577 @SuppressWarnings("checkstyle:MethodLength") 1578 public void testVersionLimits() throws Exception { 1579 final TableName tableName = TableName.valueOf(name.getMethodName()); 1580 byte [][] FAMILIES = makeNAscii(FAMILY, 3); 1581 int [] LIMITS = {1,3,5}; 1582 long [] STAMPS = makeStamps(10); 1583 byte [][] VALUES = makeNAscii(VALUE, 10); 1584 try (Table ht = TEST_UTIL.createTable(tableName, FAMILIES, LIMITS)) { 1585 1586 // Insert limit + 1 on each family 1587 Put put = new Put(ROW); 1588 put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], VALUES[0]); 1589 put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[1], VALUES[1]); 1590 put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], VALUES[0]); 1591 put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[1], VALUES[1]); 1592 put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[2], VALUES[2]); 1593 put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[3], VALUES[3]); 1594 put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[0], VALUES[0]); 1595 put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[1], VALUES[1]); 1596 put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[2], VALUES[2]); 1597 put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[3], VALUES[3]); 1598 put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[4], VALUES[4]); 1599 put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[5], VALUES[5]); 1600 put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[6], VALUES[6]); 1601 ht.put(put); 1602 1603 // Verify we only get the right number out of each 1604 1605 // Family0 1606 1607 Get get = new Get(ROW); 1608 get.addColumn(FAMILIES[0], QUALIFIER); 1609 get.setMaxVersions(Integer.MAX_VALUE); 1610 Result result = ht.get(get); 1611 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, 1612 new long [] {STAMPS[1]}, 1613 new byte[][] {VALUES[1]}, 1614 0, 0); 1615 1616 get = new Get(ROW); 1617 get.addFamily(FAMILIES[0]); 1618 get.setMaxVersions(Integer.MAX_VALUE); 1619 result = ht.get(get); 1620 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, 1621 new long [] {STAMPS[1]}, 1622 new byte[][] {VALUES[1]}, 1623 0, 0); 1624 1625 Scan scan = new Scan(ROW); 1626 scan.addColumn(FAMILIES[0], QUALIFIER); 1627 scan.setMaxVersions(Integer.MAX_VALUE); 1628 result = getSingleScanResult(ht, scan); 1629 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, 1630 new long[]{STAMPS[1]}, 1631 new byte[][]{VALUES[1]}, 1632 0, 0); 1633 1634 scan = new Scan(ROW); 1635 scan.addFamily(FAMILIES[0]); 1636 scan.setMaxVersions(Integer.MAX_VALUE); 1637 result = getSingleScanResult(ht, scan); 1638 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, 1639 new long[]{STAMPS[1]}, 1640 new byte[][]{VALUES[1]}, 1641 0, 0); 1642 1643 // Family1 1644 1645 get = new Get(ROW); 1646 get.addColumn(FAMILIES[1], QUALIFIER); 1647 get.setMaxVersions(Integer.MAX_VALUE); 1648 result = ht.get(get); 1649 assertNResult(result, ROW, FAMILIES[1], QUALIFIER, 1650 new long [] {STAMPS[1], STAMPS[2], STAMPS[3]}, 1651 new byte[][] {VALUES[1], VALUES[2], VALUES[3]}, 1652 0, 2); 1653 1654 get = new Get(ROW); 1655 get.addFamily(FAMILIES[1]); 1656 get.setMaxVersions(Integer.MAX_VALUE); 1657 result = ht.get(get); 1658 assertNResult(result, ROW, FAMILIES[1], QUALIFIER, 1659 new long [] {STAMPS[1], STAMPS[2], STAMPS[3]}, 1660 new byte[][] {VALUES[1], VALUES[2], VALUES[3]}, 1661 0, 2); 1662 1663 scan = new Scan(ROW); 1664 scan.addColumn(FAMILIES[1], QUALIFIER); 1665 scan.setMaxVersions(Integer.MAX_VALUE); 1666 result = getSingleScanResult(ht, scan); 1667 assertNResult(result, ROW, FAMILIES[1], QUALIFIER, 1668 new long[]{STAMPS[1], STAMPS[2], STAMPS[3]}, 1669 new byte[][]{VALUES[1], VALUES[2], VALUES[3]}, 1670 0, 2); 1671 1672 scan = new Scan(ROW); 1673 scan.addFamily(FAMILIES[1]); 1674 scan.setMaxVersions(Integer.MAX_VALUE); 1675 result = getSingleScanResult(ht, scan); 1676 assertNResult(result, ROW, FAMILIES[1], QUALIFIER, 1677 new long[]{STAMPS[1], STAMPS[2], STAMPS[3]}, 1678 new byte[][]{VALUES[1], VALUES[2], VALUES[3]}, 1679 0, 2); 1680 1681 // Family2 1682 1683 get = new Get(ROW); 1684 get.addColumn(FAMILIES[2], QUALIFIER); 1685 get.setMaxVersions(Integer.MAX_VALUE); 1686 result = ht.get(get); 1687 assertNResult(result, ROW, FAMILIES[2], QUALIFIER, 1688 new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]}, 1689 new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]}, 1690 0, 4); 1691 1692 get = new Get(ROW); 1693 get.addFamily(FAMILIES[2]); 1694 get.setMaxVersions(Integer.MAX_VALUE); 1695 result = ht.get(get); 1696 assertNResult(result, ROW, FAMILIES[2], QUALIFIER, 1697 new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]}, 1698 new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]}, 1699 0, 4); 1700 1701 scan = new Scan(ROW); 1702 scan.addColumn(FAMILIES[2], QUALIFIER); 1703 scan.setMaxVersions(Integer.MAX_VALUE); 1704 result = getSingleScanResult(ht, scan); 1705 assertNResult(result, ROW, FAMILIES[2], QUALIFIER, 1706 new long[]{STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]}, 1707 new byte[][]{VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]}, 1708 0, 4); 1709 1710 scan = new Scan(ROW); 1711 scan.addFamily(FAMILIES[2]); 1712 scan.setMaxVersions(Integer.MAX_VALUE); 1713 result = getSingleScanResult(ht, scan); 1714 assertNResult(result, ROW, FAMILIES[2], QUALIFIER, 1715 new long[]{STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]}, 1716 new byte[][]{VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]}, 1717 0, 4); 1718 1719 // Try all families 1720 1721 get = new Get(ROW); 1722 get.setMaxVersions(Integer.MAX_VALUE); 1723 result = ht.get(get); 1724 assertTrue("Expected 9 keys but received " + result.size(), 1725 result.size() == 9); 1726 1727 get = new Get(ROW); 1728 get.addFamily(FAMILIES[0]); 1729 get.addFamily(FAMILIES[1]); 1730 get.addFamily(FAMILIES[2]); 1731 get.setMaxVersions(Integer.MAX_VALUE); 1732 result = ht.get(get); 1733 assertTrue("Expected 9 keys but received " + result.size(), 1734 result.size() == 9); 1735 1736 get = new Get(ROW); 1737 get.addColumn(FAMILIES[0], QUALIFIER); 1738 get.addColumn(FAMILIES[1], QUALIFIER); 1739 get.addColumn(FAMILIES[2], QUALIFIER); 1740 get.setMaxVersions(Integer.MAX_VALUE); 1741 result = ht.get(get); 1742 assertTrue("Expected 9 keys but received " + result.size(), 1743 result.size() == 9); 1744 1745 scan = new Scan(ROW); 1746 scan.setMaxVersions(Integer.MAX_VALUE); 1747 result = getSingleScanResult(ht, scan); 1748 assertTrue("Expected 9 keys but received " + result.size(), 1749 result.size() == 9); 1750 1751 scan = new Scan(ROW); 1752 scan.setMaxVersions(Integer.MAX_VALUE); 1753 scan.addFamily(FAMILIES[0]); 1754 scan.addFamily(FAMILIES[1]); 1755 scan.addFamily(FAMILIES[2]); 1756 result = getSingleScanResult(ht, scan); 1757 assertTrue("Expected 9 keys but received " + result.size(), 1758 result.size() == 9); 1759 1760 scan = new Scan(ROW); 1761 scan.setMaxVersions(Integer.MAX_VALUE); 1762 scan.addColumn(FAMILIES[0], QUALIFIER); 1763 scan.addColumn(FAMILIES[1], QUALIFIER); 1764 scan.addColumn(FAMILIES[2], QUALIFIER); 1765 result = getSingleScanResult(ht, scan); 1766 assertTrue("Expected 9 keys but received " + result.size(), 1767 result.size() == 9); 1768 } 1769 } 1770 1771 @Test 1772 public void testDeleteFamilyVersion() throws Exception { 1773 try (Admin admin = TEST_UTIL.getAdmin()) { 1774 final TableName tableName = TableName.valueOf(name.getMethodName()); 1775 1776 byte[][] QUALIFIERS = makeNAscii(QUALIFIER, 1); 1777 byte[][] VALUES = makeN(VALUE, 5); 1778 long[] ts = {1000, 2000, 3000, 4000, 5000}; 1779 1780 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5)) { 1781 1782 Put put = new Put(ROW); 1783 for (int q = 0; q < 1; q++) { 1784 for (int t = 0; t < 5; t++) { 1785 put.addColumn(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]); 1786 } 1787 } 1788 ht.put(put); 1789 admin.flush(tableName); 1790 1791 Delete delete = new Delete(ROW); 1792 delete.addFamilyVersion(FAMILY, ts[1]); // delete version '2000' 1793 delete.addFamilyVersion(FAMILY, ts[3]); // delete version '4000' 1794 ht.delete(delete); 1795 admin.flush(tableName); 1796 1797 for (int i = 0; i < 1; i++) { 1798 Get get = new Get(ROW); 1799 get.addColumn(FAMILY, QUALIFIERS[i]); 1800 get.setMaxVersions(Integer.MAX_VALUE); 1801 Result result = ht.get(get); 1802 // verify version '1000'/'3000'/'5000' remains for all columns 1803 assertNResult(result, ROW, FAMILY, QUALIFIERS[i], 1804 new long[]{ts[0], ts[2], ts[4]}, 1805 new byte[][]{VALUES[0], VALUES[2], VALUES[4]}, 1806 0, 2); 1807 } 1808 } 1809 } 1810 } 1811 1812 @Test 1813 public void testDeleteFamilyVersionWithOtherDeletes() throws Exception { 1814 final TableName tableName = TableName.valueOf(name.getMethodName()); 1815 1816 byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 5); 1817 byte [][] VALUES = makeN(VALUE, 5); 1818 long [] ts = {1000, 2000, 3000, 4000, 5000}; 1819 1820 try (Admin admin = TEST_UTIL.getAdmin(); 1821 Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5)) { 1822 Put put = null; 1823 Result result = null; 1824 Get get = null; 1825 Delete delete = null; 1826 1827 // 1. put on ROW 1828 put = new Put(ROW); 1829 for (int q = 0; q < 5; q++) { 1830 for (int t = 0; t < 5; t++) { 1831 put.addColumn(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]); 1832 } 1833 } 1834 ht.put(put); 1835 admin.flush(tableName); 1836 1837 // 2. put on ROWS[0] 1838 byte[] ROW2 = Bytes.toBytes("myRowForTest"); 1839 put = new Put(ROW2); 1840 for (int q = 0; q < 5; q++) { 1841 for (int t = 0; t < 5; t++) { 1842 put.addColumn(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]); 1843 } 1844 } 1845 ht.put(put); 1846 admin.flush(tableName); 1847 1848 // 3. delete on ROW 1849 delete = new Delete(ROW); 1850 // delete version <= 2000 of all columns 1851 // note: addFamily must be the first since it will mask 1852 // the subsequent other type deletes! 1853 delete.addFamily(FAMILY, ts[1]); 1854 // delete version '4000' of all columns 1855 delete.addFamilyVersion(FAMILY, ts[3]); 1856 // delete version <= 3000 of column 0 1857 delete.addColumns(FAMILY, QUALIFIERS[0], ts[2]); 1858 // delete version <= 5000 of column 2 1859 delete.addColumns(FAMILY, QUALIFIERS[2], ts[4]); 1860 // delete version 5000 of column 4 1861 delete.addColumn(FAMILY, QUALIFIERS[4], ts[4]); 1862 ht.delete(delete); 1863 admin.flush(tableName); 1864 1865 // 4. delete on ROWS[0] 1866 delete = new Delete(ROW2); 1867 delete.addFamilyVersion(FAMILY, ts[1]); // delete version '2000' 1868 delete.addFamilyVersion(FAMILY, ts[3]); // delete version '4000' 1869 ht.delete(delete); 1870 admin.flush(tableName); 1871 1872 // 5. check ROW 1873 get = new Get(ROW); 1874 get.addColumn(FAMILY, QUALIFIERS[0]); 1875 get.setMaxVersions(Integer.MAX_VALUE); 1876 result = ht.get(get); 1877 assertNResult(result, ROW, FAMILY, QUALIFIERS[0], 1878 new long[]{ts[4]}, 1879 new byte[][]{VALUES[4]}, 1880 0, 0); 1881 1882 get = new Get(ROW); 1883 get.addColumn(FAMILY, QUALIFIERS[1]); 1884 get.setMaxVersions(Integer.MAX_VALUE); 1885 result = ht.get(get); 1886 assertNResult(result, ROW, FAMILY, QUALIFIERS[1], 1887 new long[]{ts[2], ts[4]}, 1888 new byte[][]{VALUES[2], VALUES[4]}, 1889 0, 1); 1890 1891 get = new Get(ROW); 1892 get.addColumn(FAMILY, QUALIFIERS[2]); 1893 get.setMaxVersions(Integer.MAX_VALUE); 1894 result = ht.get(get); 1895 assertEquals(0, result.size()); 1896 1897 get = new Get(ROW); 1898 get.addColumn(FAMILY, QUALIFIERS[3]); 1899 get.setMaxVersions(Integer.MAX_VALUE); 1900 result = ht.get(get); 1901 assertNResult(result, ROW, FAMILY, QUALIFIERS[3], 1902 new long[]{ts[2], ts[4]}, 1903 new byte[][]{VALUES[2], VALUES[4]}, 1904 0, 1); 1905 1906 get = new Get(ROW); 1907 get.addColumn(FAMILY, QUALIFIERS[4]); 1908 get.setMaxVersions(Integer.MAX_VALUE); 1909 result = ht.get(get); 1910 assertNResult(result, ROW, FAMILY, QUALIFIERS[4], 1911 new long[]{ts[2]}, 1912 new byte[][]{VALUES[2]}, 1913 0, 0); 1914 1915 // 6. check ROWS[0] 1916 for (int i = 0; i < 5; i++) { 1917 get = new Get(ROW2); 1918 get.addColumn(FAMILY, QUALIFIERS[i]); 1919 get.readVersions(Integer.MAX_VALUE); 1920 result = ht.get(get); 1921 // verify version '1000'/'3000'/'5000' remains for all columns 1922 assertNResult(result, ROW2, FAMILY, QUALIFIERS[i], 1923 new long[]{ts[0], ts[2], ts[4]}, 1924 new byte[][]{VALUES[0], VALUES[2], VALUES[4]}, 1925 0, 2); 1926 } 1927 } 1928 } 1929 1930 @Test 1931 public void testDeleteWithFailed() throws Exception { 1932 final TableName tableName = TableName.valueOf(name.getMethodName()); 1933 1934 byte [][] FAMILIES = makeNAscii(FAMILY, 3); 1935 byte [][] VALUES = makeN(VALUE, 5); 1936 long [] ts = {1000, 2000, 3000, 4000, 5000}; 1937 1938 try (Table ht = TEST_UTIL.createTable(tableName, FAMILIES, 3)) { 1939 Put put = new Put(ROW); 1940 put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); 1941 ht.put(put); 1942 1943 // delete wrong family 1944 Delete delete = new Delete(ROW); 1945 delete.addFamily(FAMILIES[1], ts[0]); 1946 ht.delete(delete); 1947 1948 Get get = new Get(ROW); 1949 get.addFamily(FAMILIES[0]); 1950 get.readAllVersions(); 1951 Result result = ht.get(get); 1952 assertTrue(Bytes.equals(result.getValue(FAMILIES[0], QUALIFIER), VALUES[0])); 1953 } 1954 } 1955 1956 @Test 1957 public void testDeletes() throws Exception { 1958 final TableName tableName = TableName.valueOf(name.getMethodName()); 1959 1960 byte [][] ROWS = makeNAscii(ROW, 6); 1961 byte [][] FAMILIES = makeNAscii(FAMILY, 3); 1962 byte [][] VALUES = makeN(VALUE, 5); 1963 long [] ts = {1000, 2000, 3000, 4000, 5000}; 1964 1965 try (Table ht = TEST_UTIL.createTable(tableName, FAMILIES, 3)) { 1966 1967 Put put = new Put(ROW); 1968 put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); 1969 put.addColumn(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]); 1970 ht.put(put); 1971 1972 Delete delete = new Delete(ROW); 1973 delete.addFamily(FAMILIES[0], ts[0]); 1974 ht.delete(delete); 1975 1976 Get get = new Get(ROW); 1977 get.addFamily(FAMILIES[0]); 1978 get.setMaxVersions(Integer.MAX_VALUE); 1979 Result result = ht.get(get); 1980 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, 1981 new long [] {ts[1]}, 1982 new byte[][] {VALUES[1]}, 1983 0, 0); 1984 1985 Scan scan = new Scan(ROW); 1986 scan.addFamily(FAMILIES[0]); 1987 scan.setMaxVersions(Integer.MAX_VALUE); 1988 result = getSingleScanResult(ht, scan); 1989 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, 1990 new long[]{ts[1]}, 1991 new byte[][]{VALUES[1]}, 1992 0, 0); 1993 1994 // Test delete latest version 1995 put = new Put(ROW); 1996 put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); 1997 put.addColumn(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]); 1998 put.addColumn(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]); 1999 put.addColumn(FAMILIES[0], null, ts[4], VALUES[4]); 2000 put.addColumn(FAMILIES[0], null, ts[2], VALUES[2]); 2001 put.addColumn(FAMILIES[0], null, ts[3], VALUES[3]); 2002 ht.put(put); 2003 2004 delete = new Delete(ROW); 2005 delete.addColumn(FAMILIES[0], QUALIFIER); // ts[4] 2006 ht.delete(delete); 2007 2008 get = new Get(ROW); 2009 get.addColumn(FAMILIES[0], QUALIFIER); 2010 get.setMaxVersions(Integer.MAX_VALUE); 2011 result = ht.get(get); 2012 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, 2013 new long [] {ts[1], ts[2], ts[3]}, 2014 new byte[][] {VALUES[1], VALUES[2], VALUES[3]}, 2015 0, 2); 2016 2017 scan = new Scan(ROW); 2018 scan.addColumn(FAMILIES[0], QUALIFIER); 2019 scan.setMaxVersions(Integer.MAX_VALUE); 2020 result = getSingleScanResult(ht, scan); 2021 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, 2022 new long[]{ts[1], ts[2], ts[3]}, 2023 new byte[][]{VALUES[1], VALUES[2], VALUES[3]}, 2024 0, 2); 2025 2026 // Test for HBASE-1847 2027 delete = new Delete(ROW); 2028 delete.addColumn(FAMILIES[0], null); 2029 ht.delete(delete); 2030 2031 // Cleanup null qualifier 2032 delete = new Delete(ROW); 2033 delete.addColumns(FAMILIES[0], null); 2034 ht.delete(delete); 2035 2036 // Expected client behavior might be that you can re-put deleted values 2037 // But alas, this is not to be. We can't put them back in either case. 2038 2039 put = new Put(ROW); 2040 put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000 2041 put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000 2042 ht.put(put); 2043 2044 2045 // It used to be due to the internal implementation of Get, that 2046 // the Get() call would return ts[4] UNLIKE the Scan below. With 2047 // the switch to using Scan for Get this is no longer the case. 2048 get = new Get(ROW); 2049 get.addFamily(FAMILIES[0]); 2050 get.setMaxVersions(Integer.MAX_VALUE); 2051 result = ht.get(get); 2052 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, 2053 new long [] {ts[1], ts[2], ts[3]}, 2054 new byte[][] {VALUES[1], VALUES[2], VALUES[3]}, 2055 0, 2); 2056 2057 // The Scanner returns the previous values, the expected-naive-unexpected behavior 2058 2059 scan = new Scan(ROW); 2060 scan.addFamily(FAMILIES[0]); 2061 scan.setMaxVersions(Integer.MAX_VALUE); 2062 result = getSingleScanResult(ht, scan); 2063 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, 2064 new long[]{ts[1], ts[2], ts[3]}, 2065 new byte[][]{VALUES[1], VALUES[2], VALUES[3]}, 2066 0, 2); 2067 2068 // Test deleting an entire family from one row but not the other various ways 2069 2070 put = new Put(ROWS[0]); 2071 put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]); 2072 put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]); 2073 put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]); 2074 put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]); 2075 ht.put(put); 2076 2077 put = new Put(ROWS[1]); 2078 put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]); 2079 put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]); 2080 put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]); 2081 put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]); 2082 ht.put(put); 2083 2084 put = new Put(ROWS[2]); 2085 put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]); 2086 put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]); 2087 put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]); 2088 put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]); 2089 ht.put(put); 2090 2091 // Assert that above went in. 2092 get = new Get(ROWS[2]); 2093 get.addFamily(FAMILIES[1]); 2094 get.addFamily(FAMILIES[2]); 2095 get.setMaxVersions(Integer.MAX_VALUE); 2096 result = ht.get(get); 2097 assertTrue("Expected 4 key but received " + result.size() + ": " + result, 2098 result.size() == 4); 2099 2100 delete = new Delete(ROWS[0]); 2101 delete.addFamily(FAMILIES[2]); 2102 ht.delete(delete); 2103 2104 delete = new Delete(ROWS[1]); 2105 delete.addColumns(FAMILIES[1], QUALIFIER); 2106 ht.delete(delete); 2107 2108 delete = new Delete(ROWS[2]); 2109 delete.addColumn(FAMILIES[1], QUALIFIER); 2110 delete.addColumn(FAMILIES[1], QUALIFIER); 2111 delete.addColumn(FAMILIES[2], QUALIFIER); 2112 ht.delete(delete); 2113 2114 get = new Get(ROWS[0]); 2115 get.addFamily(FAMILIES[1]); 2116 get.addFamily(FAMILIES[2]); 2117 get.setMaxVersions(Integer.MAX_VALUE); 2118 result = ht.get(get); 2119 assertTrue("Expected 2 keys but received " + result.size(), 2120 result.size() == 2); 2121 assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, 2122 new long [] {ts[0], ts[1]}, 2123 new byte[][] {VALUES[0], VALUES[1]}, 2124 0, 1); 2125 2126 scan = new Scan(ROWS[0]); 2127 scan.addFamily(FAMILIES[1]); 2128 scan.addFamily(FAMILIES[2]); 2129 scan.setMaxVersions(Integer.MAX_VALUE); 2130 result = getSingleScanResult(ht, scan); 2131 assertTrue("Expected 2 keys but received " + result.size(), 2132 result.size() == 2); 2133 assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, 2134 new long[]{ts[0], ts[1]}, 2135 new byte[][]{VALUES[0], VALUES[1]}, 2136 0, 1); 2137 2138 get = new Get(ROWS[1]); 2139 get.addFamily(FAMILIES[1]); 2140 get.addFamily(FAMILIES[2]); 2141 get.setMaxVersions(Integer.MAX_VALUE); 2142 result = ht.get(get); 2143 assertTrue("Expected 2 keys but received " + result.size(), 2144 result.size() == 2); 2145 2146 scan = new Scan(ROWS[1]); 2147 scan.addFamily(FAMILIES[1]); 2148 scan.addFamily(FAMILIES[2]); 2149 scan.setMaxVersions(Integer.MAX_VALUE); 2150 result = getSingleScanResult(ht, scan); 2151 assertTrue("Expected 2 keys but received " + result.size(), 2152 result.size() == 2); 2153 2154 get = new Get(ROWS[2]); 2155 get.addFamily(FAMILIES[1]); 2156 get.addFamily(FAMILIES[2]); 2157 get.setMaxVersions(Integer.MAX_VALUE); 2158 result = ht.get(get); 2159 assertEquals(1, result.size()); 2160 assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER, 2161 new long [] {ts[2]}, 2162 new byte[][] {VALUES[2]}, 2163 0, 0); 2164 2165 scan = new Scan(ROWS[2]); 2166 scan.addFamily(FAMILIES[1]); 2167 scan.addFamily(FAMILIES[2]); 2168 scan.setMaxVersions(Integer.MAX_VALUE); 2169 result = getSingleScanResult(ht, scan); 2170 assertEquals(1, result.size()); 2171 assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER, 2172 new long[]{ts[2]}, 2173 new byte[][]{VALUES[2]}, 2174 0, 0); 2175 2176 // Test if we delete the family first in one row (HBASE-1541) 2177 2178 delete = new Delete(ROWS[3]); 2179 delete.addFamily(FAMILIES[1]); 2180 ht.delete(delete); 2181 2182 put = new Put(ROWS[3]); 2183 put.addColumn(FAMILIES[2], QUALIFIER, VALUES[0]); 2184 ht.put(put); 2185 2186 put = new Put(ROWS[4]); 2187 put.addColumn(FAMILIES[1], QUALIFIER, VALUES[1]); 2188 put.addColumn(FAMILIES[2], QUALIFIER, VALUES[2]); 2189 ht.put(put); 2190 2191 get = new Get(ROWS[3]); 2192 get.addFamily(FAMILIES[1]); 2193 get.addFamily(FAMILIES[2]); 2194 get.setMaxVersions(Integer.MAX_VALUE); 2195 result = ht.get(get); 2196 assertTrue("Expected 1 key but received " + result.size(), 2197 result.size() == 1); 2198 2199 get = new Get(ROWS[4]); 2200 get.addFamily(FAMILIES[1]); 2201 get.addFamily(FAMILIES[2]); 2202 get.setMaxVersions(Integer.MAX_VALUE); 2203 result = ht.get(get); 2204 assertTrue("Expected 2 keys but received " + result.size(), 2205 result.size() == 2); 2206 2207 scan = new Scan(ROWS[3]); 2208 scan.addFamily(FAMILIES[1]); 2209 scan.addFamily(FAMILIES[2]); 2210 scan.setMaxVersions(Integer.MAX_VALUE); 2211 try (ResultScanner scanner = ht.getScanner(scan)) { 2212 result = scanner.next(); 2213 assertTrue("Expected 1 key but received " + result.size(), 2214 result.size() == 1); 2215 assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[3])); 2216 assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[0])); 2217 result = scanner.next(); 2218 assertTrue("Expected 2 keys but received " + result.size(), 2219 result.size() == 2); 2220 assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[4])); 2221 assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[1]), ROWS[4])); 2222 assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[1])); 2223 assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[1]), VALUES[2])); 2224 } 2225 2226 // Add test of bulk deleting. 2227 for (int i = 0; i < 10; i++) { 2228 byte[] bytes = Bytes.toBytes(i); 2229 put = new Put(bytes); 2230 put.setDurability(Durability.SKIP_WAL); 2231 put.addColumn(FAMILIES[0], QUALIFIER, bytes); 2232 ht.put(put); 2233 } 2234 for (int i = 0; i < 10; i++) { 2235 byte[] bytes = Bytes.toBytes(i); 2236 get = new Get(bytes); 2237 get.addFamily(FAMILIES[0]); 2238 result = ht.get(get); 2239 assertTrue(result.size() == 1); 2240 } 2241 ArrayList<Delete> deletes = new ArrayList<>(); 2242 for (int i = 0; i < 10; i++) { 2243 byte[] bytes = Bytes.toBytes(i); 2244 delete = new Delete(bytes); 2245 delete.addFamily(FAMILIES[0]); 2246 deletes.add(delete); 2247 } 2248 ht.delete(deletes); 2249 for (int i = 0; i < 10; i++) { 2250 byte[] bytes = Bytes.toBytes(i); 2251 get = new Get(bytes); 2252 get.addFamily(FAMILIES[0]); 2253 result = ht.get(get); 2254 assertTrue(result.isEmpty()); 2255 } 2256 } 2257 } 2258 2259 /** 2260 * Test batch operations with combination of valid and invalid args 2261 */ 2262 @Test 2263 public void testBatchOperationsWithErrors() throws Exception { 2264 final TableName tableName = TableName.valueOf(name.getMethodName()); 2265 try (Table foo = TEST_UTIL.createTable(tableName, new byte[][] {FAMILY}, 10)) { 2266 2267 int NUM_OPS = 100; 2268 int FAILED_OPS = 50; 2269 2270 RetriesExhaustedWithDetailsException expectedException = null; 2271 IllegalArgumentException iae = null; 2272 2273 // 1.1 Put with no column families (local validation, runtime exception) 2274 List<Put> puts = new ArrayList<Put>(NUM_OPS); 2275 for (int i = 0; i != NUM_OPS; i++) { 2276 Put put = new Put(Bytes.toBytes(i)); 2277 puts.add(put); 2278 } 2279 2280 try { 2281 foo.put(puts); 2282 } catch (IllegalArgumentException e) { 2283 iae = e; 2284 } 2285 assertNotNull(iae); 2286 assertEquals(NUM_OPS, puts.size()); 2287 2288 // 1.2 Put with invalid column family 2289 iae = null; 2290 puts.clear(); 2291 for (int i = 0; i != NUM_OPS; i++) { 2292 Put put = new Put(Bytes.toBytes(i)); 2293 put.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY, Bytes.toBytes(i)); 2294 puts.add(put); 2295 } 2296 2297 try { 2298 foo.put(puts); 2299 } catch (RetriesExhaustedWithDetailsException e) { 2300 expectedException = e; 2301 } 2302 assertNotNull(expectedException); 2303 assertEquals(FAILED_OPS, expectedException.exceptions.size()); 2304 assertTrue(expectedException.actions.contains(puts.get(1))); 2305 2306 // 2.1 Get non-existent rows 2307 List<Get> gets = new ArrayList<>(NUM_OPS); 2308 for (int i = 0; i < NUM_OPS; i++) { 2309 Get get = new Get(Bytes.toBytes(i)); 2310 // get.addColumn(FAMILY, FAMILY); 2311 gets.add(get); 2312 } 2313 Result[] getsResult = foo.get(gets); 2314 2315 assertNotNull(getsResult); 2316 assertEquals(NUM_OPS, getsResult.length); 2317 assertNull(getsResult[1].getRow()); 2318 2319 // 2.2 Get with invalid column family 2320 gets.clear(); 2321 getsResult = null; 2322 expectedException = null; 2323 for (int i = 0; i < NUM_OPS; i++) { 2324 Get get = new Get(Bytes.toBytes(i)); 2325 get.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY); 2326 gets.add(get); 2327 } 2328 try { 2329 getsResult = foo.get(gets); 2330 } catch (RetriesExhaustedWithDetailsException e) { 2331 expectedException = e; 2332 } 2333 assertNull(getsResult); 2334 assertNotNull(expectedException); 2335 assertEquals(FAILED_OPS, expectedException.exceptions.size()); 2336 assertTrue(expectedException.actions.contains(gets.get(1))); 2337 2338 // 3.1 Delete with invalid column family 2339 expectedException = null; 2340 List<Delete> deletes = new ArrayList<>(NUM_OPS); 2341 for (int i = 0; i < NUM_OPS; i++) { 2342 Delete delete = new Delete(Bytes.toBytes(i)); 2343 delete.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY); 2344 deletes.add(delete); 2345 } 2346 try { 2347 foo.delete(deletes); 2348 } catch (RetriesExhaustedWithDetailsException e) { 2349 expectedException = e; 2350 } 2351 assertEquals((NUM_OPS - FAILED_OPS), deletes.size()); 2352 assertNotNull(expectedException); 2353 assertEquals(FAILED_OPS, expectedException.exceptions.size()); 2354 assertTrue(expectedException.actions.contains(deletes.get(1))); 2355 2356 2357 // 3.2 Delete non-existent rows 2358 deletes.clear(); 2359 for (int i = 0; i < NUM_OPS; i++) { 2360 Delete delete = new Delete(Bytes.toBytes(i)); 2361 deletes.add(delete); 2362 } 2363 foo.delete(deletes); 2364 2365 assertTrue(deletes.isEmpty()); 2366 } 2367 } 2368 2369 /* 2370 * Baseline "scalability" test. 2371 * 2372 * Tests one hundred families, one million columns, one million versions 2373 */ 2374 @Ignore @Test 2375 public void testMillions() throws Exception { 2376 2377 // 100 families 2378 2379 // millions of columns 2380 2381 // millions of versions 2382 2383 } 2384 2385 @Ignore @Test 2386 public void testMultipleRegionsAndBatchPuts() throws Exception { 2387 // Two family table 2388 2389 // Insert lots of rows 2390 2391 // Insert to the same row with batched puts 2392 2393 // Insert to multiple rows with batched puts 2394 2395 // Split the table 2396 2397 // Get row from first region 2398 2399 // Get row from second region 2400 2401 // Scan all rows 2402 2403 // Insert to multiple regions with batched puts 2404 2405 // Get row from first region 2406 2407 // Get row from second region 2408 2409 // Scan all rows 2410 2411 2412 } 2413 2414 @Ignore @Test 2415 public void testMultipleRowMultipleFamily() throws Exception { 2416 2417 } 2418 2419 // 2420 // JIRA Testers 2421 // 2422 2423 /** 2424 * HBASE-867 2425 * If millions of columns in a column family, hbase scanner won't come up 2426 * 2427 * Test will create numRows rows, each with numColsPerRow columns 2428 * (1 version each), and attempt to scan them all. 2429 * 2430 * To test at scale, up numColsPerRow to the millions 2431 * (have not gotten that to work running as junit though) 2432 */ 2433 @Test 2434 public void testJiraTest867() throws Exception { 2435 int numRows = 10; 2436 int numColsPerRow = 2000; 2437 2438 final TableName tableName = TableName.valueOf(name.getMethodName()); 2439 2440 byte [][] ROWS = makeN(ROW, numRows); 2441 byte [][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow); 2442 2443 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 2444 2445 // Insert rows 2446 2447 for (int i = 0; i < numRows; i++) { 2448 Put put = new Put(ROWS[i]); 2449 put.setDurability(Durability.SKIP_WAL); 2450 for (int j = 0; j < numColsPerRow; j++) { 2451 put.addColumn(FAMILY, QUALIFIERS[j], QUALIFIERS[j]); 2452 } 2453 assertTrue("Put expected to contain " + numColsPerRow + " columns but " + 2454 "only contains " + put.size(), put.size() == numColsPerRow); 2455 ht.put(put); 2456 } 2457 2458 // Get a row 2459 Get get = new Get(ROWS[numRows - 1]); 2460 Result result = ht.get(get); 2461 assertNumKeys(result, numColsPerRow); 2462 Cell[] keys = result.rawCells(); 2463 for (int i = 0; i < result.size(); i++) { 2464 assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]); 2465 } 2466 2467 // Scan the rows 2468 Scan scan = new Scan(); 2469 try (ResultScanner scanner = ht.getScanner(scan)) { 2470 int rowCount = 0; 2471 while ((result = scanner.next()) != null) { 2472 assertNumKeys(result, numColsPerRow); 2473 Cell[] kvs = result.rawCells(); 2474 for (int i = 0; i < numColsPerRow; i++) { 2475 assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]); 2476 } 2477 rowCount++; 2478 } 2479 assertTrue("Expected to scan " + numRows + " rows but actually scanned " 2480 + rowCount + " rows", rowCount == numRows); 2481 } 2482 2483 // flush and try again 2484 2485 TEST_UTIL.flush(); 2486 2487 // Get a row 2488 get = new Get(ROWS[numRows - 1]); 2489 result = ht.get(get); 2490 assertNumKeys(result, numColsPerRow); 2491 keys = result.rawCells(); 2492 for (int i = 0; i < result.size(); i++) { 2493 assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]); 2494 } 2495 2496 // Scan the rows 2497 scan = new Scan(); 2498 try (ResultScanner scanner = ht.getScanner(scan)) { 2499 int rowCount = 0; 2500 while ((result = scanner.next()) != null) { 2501 assertNumKeys(result, numColsPerRow); 2502 Cell[] kvs = result.rawCells(); 2503 for (int i = 0; i < numColsPerRow; i++) { 2504 assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]); 2505 } 2506 rowCount++; 2507 } 2508 assertTrue("Expected to scan " + numRows + " rows but actually scanned " 2509 + rowCount + " rows", rowCount == numRows); 2510 } 2511 } 2512 } 2513 2514 /** 2515 * HBASE-861 2516 * get with timestamp will return a value if there is a version with an 2517 * earlier timestamp 2518 */ 2519 @Test 2520 public void testJiraTest861() throws Exception { 2521 final TableName tableName = TableName.valueOf(name.getMethodName()); 2522 byte [][] VALUES = makeNAscii(VALUE, 7); 2523 long [] STAMPS = makeStamps(7); 2524 2525 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) { 2526 2527 // Insert three versions 2528 2529 Put put = new Put(ROW); 2530 put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); 2531 put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 2532 put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 2533 ht.put(put); 2534 2535 // Get the middle value 2536 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 2537 2538 // Try to get one version before (expect fail) 2539 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]); 2540 2541 // Try to get one version after (expect fail) 2542 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]); 2543 2544 // Try same from storefile 2545 TEST_UTIL.flush(); 2546 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 2547 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]); 2548 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]); 2549 2550 // Insert two more versions surrounding others, into memstore 2551 put = new Put(ROW); 2552 put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]); 2553 put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]); 2554 ht.put(put); 2555 2556 // Check we can get everything we should and can't get what we shouldn't 2557 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]); 2558 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]); 2559 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 2560 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); 2561 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 2562 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]); 2563 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]); 2564 2565 // Try same from two storefiles 2566 TEST_UTIL.flush(); 2567 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]); 2568 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]); 2569 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 2570 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); 2571 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 2572 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]); 2573 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]); 2574 } 2575 } 2576 2577 /** 2578 * HBASE-33 2579 * Add a HTable get/obtainScanner method that retrieves all versions of a 2580 * particular column and row between two timestamps 2581 */ 2582 @Test 2583 public void testJiraTest33() throws Exception { 2584 final TableName tableName = TableName.valueOf(name.getMethodName()); 2585 byte [][] VALUES = makeNAscii(VALUE, 7); 2586 long [] STAMPS = makeStamps(7); 2587 2588 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) { 2589 2590 // Insert lots versions 2591 2592 Put put = new Put(ROW); 2593 put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]); 2594 put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 2595 put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 2596 put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); 2597 put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 2598 put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 2599 ht.put(put); 2600 2601 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 2602 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2); 2603 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 2604 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3); 2605 2606 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 2607 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2); 2608 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 2609 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3); 2610 2611 // Try same from storefile 2612 TEST_UTIL.flush(); 2613 2614 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 2615 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2); 2616 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 2617 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3); 2618 2619 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 2620 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2); 2621 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 2622 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3); 2623 } 2624 } 2625 2626 /** 2627 * HBASE-1014 2628 * commit(BatchUpdate) method should return timestamp 2629 */ 2630 @Test 2631 public void testJiraTest1014() throws Exception { 2632 final TableName tableName = TableName.valueOf(name.getMethodName()); 2633 2634 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) { 2635 2636 long manualStamp = 12345; 2637 2638 // Insert lots versions 2639 2640 Put put = new Put(ROW); 2641 put.addColumn(FAMILY, QUALIFIER, manualStamp, VALUE); 2642 ht.put(put); 2643 2644 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE); 2645 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp - 1); 2646 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp + 1); 2647 } 2648 } 2649 2650 /** 2651 * HBASE-1182 2652 * Scan for columns > some timestamp 2653 */ 2654 @Test 2655 public void testJiraTest1182() throws Exception { 2656 final TableName tableName = TableName.valueOf(name.getMethodName()); 2657 byte [][] VALUES = makeNAscii(VALUE, 7); 2658 long [] STAMPS = makeStamps(7); 2659 2660 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) { 2661 2662 // Insert lots versions 2663 2664 Put put = new Put(ROW); 2665 put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]); 2666 put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 2667 put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 2668 put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); 2669 put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 2670 put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 2671 ht.put(put); 2672 2673 getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 2674 getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5); 2675 getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 2676 2677 scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 2678 scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5); 2679 scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 2680 2681 // Try same from storefile 2682 TEST_UTIL.flush(); 2683 2684 getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 2685 getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5); 2686 getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 2687 2688 scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 2689 scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5); 2690 scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 2691 } 2692 } 2693 2694 /** 2695 * HBASE-52 2696 * Add a means of scanning over all versions 2697 */ 2698 @Test 2699 public void testJiraTest52() throws Exception { 2700 final TableName tableName = TableName.valueOf(name.getMethodName()); 2701 byte [][] VALUES = makeNAscii(VALUE, 7); 2702 long [] STAMPS = makeStamps(7); 2703 2704 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) { 2705 2706 // Insert lots versions 2707 2708 Put put = new Put(ROW); 2709 put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]); 2710 put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 2711 put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 2712 put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); 2713 put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 2714 put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 2715 ht.put(put); 2716 2717 getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 2718 2719 scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 2720 2721 // Try same from storefile 2722 TEST_UTIL.flush(); 2723 2724 getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 2725 2726 scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 2727 } 2728 } 2729 2730 // 2731 // Bulk Testers 2732 // 2733 2734 private void getVersionRangeAndVerifyGreaterThan(Table ht, byte [] row, 2735 byte [] family, byte [] qualifier, long [] stamps, byte [][] values, 2736 int start, int end) 2737 throws IOException { 2738 Get get = new Get(row); 2739 get.addColumn(family, qualifier); 2740 get.setMaxVersions(Integer.MAX_VALUE); 2741 get.setTimeRange(stamps[start+1], Long.MAX_VALUE); 2742 Result result = ht.get(get); 2743 assertNResult(result, row, family, qualifier, stamps, values, start+1, end); 2744 } 2745 2746 private void getVersionRangeAndVerify(Table ht, byte [] row, byte [] family, 2747 byte [] qualifier, long [] stamps, byte [][] values, int start, int end) 2748 throws IOException { 2749 Get get = new Get(row); 2750 get.addColumn(family, qualifier); 2751 get.setMaxVersions(Integer.MAX_VALUE); 2752 get.setTimeRange(stamps[start], stamps[end]+1); 2753 Result result = ht.get(get); 2754 assertNResult(result, row, family, qualifier, stamps, values, start, end); 2755 } 2756 2757 private void getAllVersionsAndVerify(Table ht, byte [] row, byte [] family, 2758 byte [] qualifier, long [] stamps, byte [][] values, int start, int end) 2759 throws IOException { 2760 Get get = new Get(row); 2761 get.addColumn(family, qualifier); 2762 get.setMaxVersions(Integer.MAX_VALUE); 2763 Result result = ht.get(get); 2764 assertNResult(result, row, family, qualifier, stamps, values, start, end); 2765 } 2766 2767 private void scanVersionRangeAndVerifyGreaterThan(Table ht, byte [] row, 2768 byte [] family, byte [] qualifier, long [] stamps, byte [][] values, 2769 int start, int end) 2770 throws IOException { 2771 Scan scan = new Scan(row); 2772 scan.addColumn(family, qualifier); 2773 scan.setMaxVersions(Integer.MAX_VALUE); 2774 scan.setTimeRange(stamps[start+1], Long.MAX_VALUE); 2775 Result result = getSingleScanResult(ht, scan); 2776 assertNResult(result, row, family, qualifier, stamps, values, start+1, end); 2777 } 2778 2779 private void scanVersionRangeAndVerify(Table ht, byte [] row, byte [] family, 2780 byte [] qualifier, long [] stamps, byte [][] values, int start, int end) 2781 throws IOException { 2782 Scan scan = new Scan(row); 2783 scan.addColumn(family, qualifier); 2784 scan.setMaxVersions(Integer.MAX_VALUE); 2785 scan.setTimeRange(stamps[start], stamps[end]+1); 2786 Result result = getSingleScanResult(ht, scan); 2787 assertNResult(result, row, family, qualifier, stamps, values, start, end); 2788 } 2789 2790 private void scanAllVersionsAndVerify(Table ht, byte [] row, byte [] family, 2791 byte [] qualifier, long [] stamps, byte [][] values, int start, int end) 2792 throws IOException { 2793 Scan scan = new Scan(row); 2794 scan.addColumn(family, qualifier); 2795 scan.setMaxVersions(Integer.MAX_VALUE); 2796 Result result = getSingleScanResult(ht, scan); 2797 assertNResult(result, row, family, qualifier, stamps, values, start, end); 2798 } 2799 2800 private void getVersionAndVerify(Table ht, byte [] row, byte [] family, 2801 byte [] qualifier, long stamp, byte [] value) 2802 throws Exception { 2803 Get get = new Get(row); 2804 get.addColumn(family, qualifier); 2805 get.setTimestamp(stamp); 2806 get.setMaxVersions(Integer.MAX_VALUE); 2807 Result result = ht.get(get); 2808 assertSingleResult(result, row, family, qualifier, stamp, value); 2809 } 2810 2811 private void getVersionAndVerifyMissing(Table ht, byte [] row, byte [] family, 2812 byte [] qualifier, long stamp) 2813 throws Exception { 2814 Get get = new Get(row); 2815 get.addColumn(family, qualifier); 2816 get.setTimestamp(stamp); 2817 get.setMaxVersions(Integer.MAX_VALUE); 2818 Result result = ht.get(get); 2819 assertEmptyResult(result); 2820 } 2821 2822 private void scanVersionAndVerify(Table ht, byte [] row, byte [] family, 2823 byte [] qualifier, long stamp, byte [] value) 2824 throws Exception { 2825 Scan scan = new Scan(row); 2826 scan.addColumn(family, qualifier); 2827 scan.setTimestamp(stamp); 2828 scan.setMaxVersions(Integer.MAX_VALUE); 2829 Result result = getSingleScanResult(ht, scan); 2830 assertSingleResult(result, row, family, qualifier, stamp, value); 2831 } 2832 2833 private void scanVersionAndVerifyMissing(Table ht, byte [] row, 2834 byte [] family, byte [] qualifier, long stamp) 2835 throws Exception { 2836 Scan scan = new Scan(row); 2837 scan.addColumn(family, qualifier); 2838 scan.setTimestamp(stamp); 2839 scan.setMaxVersions(Integer.MAX_VALUE); 2840 Result result = getSingleScanResult(ht, scan); 2841 assertNullResult(result); 2842 } 2843 2844 private void getTestNull(Table ht, byte [] row, byte [] family, 2845 byte [] value) 2846 throws Exception { 2847 2848 Get get = new Get(row); 2849 get.addColumn(family, null); 2850 Result result = ht.get(get); 2851 assertSingleResult(result, row, family, null, value); 2852 2853 get = new Get(row); 2854 get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY); 2855 result = ht.get(get); 2856 assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); 2857 2858 get = new Get(row); 2859 get.addFamily(family); 2860 result = ht.get(get); 2861 assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); 2862 2863 get = new Get(row); 2864 result = ht.get(get); 2865 assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); 2866 2867 } 2868 2869 private void getTestNull(Table ht, byte[] row, byte[] family, long value) throws Exception { 2870 Get get = new Get(row); 2871 get.addColumn(family, null); 2872 Result result = ht.get(get); 2873 assertSingleResult(result, row, family, null, value); 2874 2875 get = new Get(row); 2876 get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY); 2877 result = ht.get(get); 2878 assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); 2879 2880 get = new Get(row); 2881 get.addFamily(family); 2882 result = ht.get(get); 2883 assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); 2884 2885 get = new Get(row); 2886 result = ht.get(get); 2887 assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); 2888 } 2889 2890 private void scanTestNull(Table ht, byte[] row, byte[] family, byte[] value) 2891 throws Exception { 2892 scanTestNull(ht, row, family, value, false); 2893 } 2894 2895 private void scanTestNull(Table ht, byte[] row, byte[] family, byte[] value, 2896 boolean isReversedScan) throws Exception { 2897 2898 Scan scan = new Scan(); 2899 scan.setReversed(isReversedScan); 2900 scan.addColumn(family, null); 2901 Result result = getSingleScanResult(ht, scan); 2902 assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); 2903 2904 scan = new Scan(); 2905 scan.setReversed(isReversedScan); 2906 scan.addColumn(family, HConstants.EMPTY_BYTE_ARRAY); 2907 result = getSingleScanResult(ht, scan); 2908 assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); 2909 2910 scan = new Scan(); 2911 scan.setReversed(isReversedScan); 2912 scan.addFamily(family); 2913 result = getSingleScanResult(ht, scan); 2914 assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); 2915 2916 scan = new Scan(); 2917 scan.setReversed(isReversedScan); 2918 result = getSingleScanResult(ht, scan); 2919 assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value); 2920 2921 } 2922 2923 private void singleRowGetTest(Table ht, byte [][] ROWS, byte [][] FAMILIES, 2924 byte [][] QUALIFIERS, byte [][] VALUES) 2925 throws Exception { 2926 2927 // Single column from memstore 2928 Get get = new Get(ROWS[0]); 2929 get.addColumn(FAMILIES[4], QUALIFIERS[0]); 2930 Result result = ht.get(get); 2931 assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]); 2932 2933 // Single column from storefile 2934 get = new Get(ROWS[0]); 2935 get.addColumn(FAMILIES[2], QUALIFIERS[2]); 2936 result = ht.get(get); 2937 assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]); 2938 2939 // Single column from storefile, family match 2940 get = new Get(ROWS[0]); 2941 get.addFamily(FAMILIES[7]); 2942 result = ht.get(get); 2943 assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]); 2944 2945 // Two columns, one from memstore one from storefile, same family, 2946 // wildcard match 2947 get = new Get(ROWS[0]); 2948 get.addFamily(FAMILIES[4]); 2949 result = ht.get(get); 2950 assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0], 2951 FAMILIES[4], QUALIFIERS[4], VALUES[4]); 2952 2953 // Two columns, one from memstore one from storefile, same family, 2954 // explicit match 2955 get = new Get(ROWS[0]); 2956 get.addColumn(FAMILIES[4], QUALIFIERS[0]); 2957 get.addColumn(FAMILIES[4], QUALIFIERS[4]); 2958 result = ht.get(get); 2959 assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0], 2960 FAMILIES[4], QUALIFIERS[4], VALUES[4]); 2961 2962 // Three column, one from memstore two from storefile, different families, 2963 // wildcard match 2964 get = new Get(ROWS[0]); 2965 get.addFamily(FAMILIES[4]); 2966 get.addFamily(FAMILIES[7]); 2967 result = ht.get(get); 2968 assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, 2969 new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} }); 2970 2971 // Multiple columns from everywhere storefile, many family, wildcard 2972 get = new Get(ROWS[0]); 2973 get.addFamily(FAMILIES[2]); 2974 get.addFamily(FAMILIES[4]); 2975 get.addFamily(FAMILIES[6]); 2976 get.addFamily(FAMILIES[7]); 2977 result = ht.get(get); 2978 assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, 2979 new int [][] { 2980 {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7} 2981 }); 2982 2983 // Multiple columns from everywhere storefile, many family, wildcard 2984 get = new Get(ROWS[0]); 2985 get.addColumn(FAMILIES[2], QUALIFIERS[2]); 2986 get.addColumn(FAMILIES[2], QUALIFIERS[4]); 2987 get.addColumn(FAMILIES[4], QUALIFIERS[0]); 2988 get.addColumn(FAMILIES[4], QUALIFIERS[4]); 2989 get.addColumn(FAMILIES[6], QUALIFIERS[6]); 2990 get.addColumn(FAMILIES[6], QUALIFIERS[7]); 2991 get.addColumn(FAMILIES[7], QUALIFIERS[7]); 2992 get.addColumn(FAMILIES[7], QUALIFIERS[8]); 2993 result = ht.get(get); 2994 assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, 2995 new int [][] { 2996 {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7} 2997 }); 2998 2999 // Everything 3000 get = new Get(ROWS[0]); 3001 result = ht.get(get); 3002 assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, 3003 new int [][] { 3004 {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0} 3005 }); 3006 3007 // Get around inserted columns 3008 3009 get = new Get(ROWS[1]); 3010 result = ht.get(get); 3011 assertEmptyResult(result); 3012 3013 get = new Get(ROWS[0]); 3014 get.addColumn(FAMILIES[4], QUALIFIERS[3]); 3015 get.addColumn(FAMILIES[2], QUALIFIERS[3]); 3016 result = ht.get(get); 3017 assertEmptyResult(result); 3018 3019 } 3020 3021 private void singleRowScanTest(Table ht, byte [][] ROWS, byte [][] FAMILIES, 3022 byte [][] QUALIFIERS, byte [][] VALUES) 3023 throws Exception { 3024 3025 // Single column from memstore 3026 Scan scan = new Scan(); 3027 scan.addColumn(FAMILIES[4], QUALIFIERS[0]); 3028 Result result = getSingleScanResult(ht, scan); 3029 assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]); 3030 3031 // Single column from storefile 3032 scan = new Scan(); 3033 scan.addColumn(FAMILIES[2], QUALIFIERS[2]); 3034 result = getSingleScanResult(ht, scan); 3035 assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]); 3036 3037 // Single column from storefile, family match 3038 scan = new Scan(); 3039 scan.addFamily(FAMILIES[7]); 3040 result = getSingleScanResult(ht, scan); 3041 assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]); 3042 3043 // Two columns, one from memstore one from storefile, same family, 3044 // wildcard match 3045 scan = new Scan(); 3046 scan.addFamily(FAMILIES[4]); 3047 result = getSingleScanResult(ht, scan); 3048 assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0], 3049 FAMILIES[4], QUALIFIERS[4], VALUES[4]); 3050 3051 // Two columns, one from memstore one from storefile, same family, 3052 // explicit match 3053 scan = new Scan(); 3054 scan.addColumn(FAMILIES[4], QUALIFIERS[0]); 3055 scan.addColumn(FAMILIES[4], QUALIFIERS[4]); 3056 result = getSingleScanResult(ht, scan); 3057 assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0], 3058 FAMILIES[4], QUALIFIERS[4], VALUES[4]); 3059 3060 // Three column, one from memstore two from storefile, different families, 3061 // wildcard match 3062 scan = new Scan(); 3063 scan.addFamily(FAMILIES[4]); 3064 scan.addFamily(FAMILIES[7]); 3065 result = getSingleScanResult(ht, scan); 3066 assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, 3067 new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} }); 3068 3069 // Multiple columns from everywhere storefile, many family, wildcard 3070 scan = new Scan(); 3071 scan.addFamily(FAMILIES[2]); 3072 scan.addFamily(FAMILIES[4]); 3073 scan.addFamily(FAMILIES[6]); 3074 scan.addFamily(FAMILIES[7]); 3075 result = getSingleScanResult(ht, scan); 3076 assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, 3077 new int [][] { 3078 {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7} 3079 }); 3080 3081 // Multiple columns from everywhere storefile, many family, wildcard 3082 scan = new Scan(); 3083 scan.addColumn(FAMILIES[2], QUALIFIERS[2]); 3084 scan.addColumn(FAMILIES[2], QUALIFIERS[4]); 3085 scan.addColumn(FAMILIES[4], QUALIFIERS[0]); 3086 scan.addColumn(FAMILIES[4], QUALIFIERS[4]); 3087 scan.addColumn(FAMILIES[6], QUALIFIERS[6]); 3088 scan.addColumn(FAMILIES[6], QUALIFIERS[7]); 3089 scan.addColumn(FAMILIES[7], QUALIFIERS[7]); 3090 scan.addColumn(FAMILIES[7], QUALIFIERS[8]); 3091 result = getSingleScanResult(ht, scan); 3092 assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, 3093 new int [][] { 3094 {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7} 3095 }); 3096 3097 // Everything 3098 scan = new Scan(); 3099 result = getSingleScanResult(ht, scan); 3100 assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES, 3101 new int [][] { 3102 {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0} 3103 }); 3104 3105 // Scan around inserted columns 3106 3107 scan = new Scan(ROWS[1]); 3108 result = getSingleScanResult(ht, scan); 3109 assertNullResult(result); 3110 3111 scan = new Scan(); 3112 scan.addColumn(FAMILIES[4], QUALIFIERS[3]); 3113 scan.addColumn(FAMILIES[2], QUALIFIERS[3]); 3114 result = getSingleScanResult(ht, scan); 3115 assertNullResult(result); 3116 } 3117 3118 /** 3119 * Verify a single column using gets. 3120 * Expects family and qualifier arrays to be valid for at least 3121 * the range: idx-2 < idx < idx+2 3122 */ 3123 private void getVerifySingleColumn(Table ht, 3124 byte [][] ROWS, int ROWIDX, 3125 byte [][] FAMILIES, int FAMILYIDX, 3126 byte [][] QUALIFIERS, int QUALIFIERIDX, 3127 byte [][] VALUES, int VALUEIDX) 3128 throws Exception { 3129 3130 Get get = new Get(ROWS[ROWIDX]); 3131 Result result = ht.get(get); 3132 assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], 3133 QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); 3134 3135 get = new Get(ROWS[ROWIDX]); 3136 get.addFamily(FAMILIES[FAMILYIDX]); 3137 result = ht.get(get); 3138 assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], 3139 QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); 3140 3141 get = new Get(ROWS[ROWIDX]); 3142 get.addFamily(FAMILIES[FAMILYIDX-2]); 3143 get.addFamily(FAMILIES[FAMILYIDX]); 3144 get.addFamily(FAMILIES[FAMILYIDX+2]); 3145 result = ht.get(get); 3146 assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], 3147 QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); 3148 3149 get = new Get(ROWS[ROWIDX]); 3150 get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[0]); 3151 result = ht.get(get); 3152 assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], 3153 QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); 3154 3155 get = new Get(ROWS[ROWIDX]); 3156 get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[1]); 3157 get.addFamily(FAMILIES[FAMILYIDX]); 3158 result = ht.get(get); 3159 assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], 3160 QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); 3161 3162 get = new Get(ROWS[ROWIDX]); 3163 get.addFamily(FAMILIES[FAMILYIDX]); 3164 get.addColumn(FAMILIES[FAMILYIDX+1], QUALIFIERS[1]); 3165 get.addColumn(FAMILIES[FAMILYIDX-2], QUALIFIERS[1]); 3166 get.addFamily(FAMILIES[FAMILYIDX-1]); 3167 get.addFamily(FAMILIES[FAMILYIDX+2]); 3168 result = ht.get(get); 3169 assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], 3170 QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); 3171 3172 } 3173 3174 3175 /** 3176 * Verify a single column using scanners. 3177 * Expects family and qualifier arrays to be valid for at least 3178 * the range: idx-2 to idx+2 3179 * Expects row array to be valid for at least idx to idx+2 3180 */ 3181 private void scanVerifySingleColumn(Table ht, 3182 byte [][] ROWS, int ROWIDX, 3183 byte [][] FAMILIES, int FAMILYIDX, 3184 byte [][] QUALIFIERS, int QUALIFIERIDX, 3185 byte [][] VALUES, int VALUEIDX) 3186 throws Exception { 3187 3188 Scan scan = new Scan(); 3189 Result result = getSingleScanResult(ht, scan); 3190 assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], 3191 QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); 3192 3193 scan = new Scan(ROWS[ROWIDX]); 3194 result = getSingleScanResult(ht, scan); 3195 assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], 3196 QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); 3197 3198 scan = new Scan(ROWS[ROWIDX], ROWS[ROWIDX+1]); 3199 result = getSingleScanResult(ht, scan); 3200 assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], 3201 QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); 3202 3203 scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX+1]); 3204 result = getSingleScanResult(ht, scan); 3205 assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], 3206 QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); 3207 3208 scan = new Scan(); 3209 scan.addFamily(FAMILIES[FAMILYIDX]); 3210 result = getSingleScanResult(ht, scan); 3211 assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], 3212 QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); 3213 3214 scan = new Scan(); 3215 scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]); 3216 result = getSingleScanResult(ht, scan); 3217 assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], 3218 QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); 3219 3220 scan = new Scan(); 3221 scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]); 3222 scan.addFamily(FAMILIES[FAMILYIDX]); 3223 result = getSingleScanResult(ht, scan); 3224 assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], 3225 QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); 3226 3227 scan = new Scan(); 3228 scan.addColumn(FAMILIES[FAMILYIDX-1], QUALIFIERS[QUALIFIERIDX+1]); 3229 scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]); 3230 scan.addFamily(FAMILIES[FAMILYIDX+1]); 3231 result = getSingleScanResult(ht, scan); 3232 assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX], 3233 QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]); 3234 3235 } 3236 3237 /** 3238 * Verify we do not read any values by accident around a single column 3239 * Same requirements as getVerifySingleColumn 3240 */ 3241 private void getVerifySingleEmpty(Table ht, 3242 byte [][] ROWS, int ROWIDX, 3243 byte [][] FAMILIES, int FAMILYIDX, 3244 byte [][] QUALIFIERS, int QUALIFIERIDX) 3245 throws Exception { 3246 3247 Get get = new Get(ROWS[ROWIDX]); 3248 get.addFamily(FAMILIES[4]); 3249 get.addColumn(FAMILIES[4], QUALIFIERS[1]); 3250 Result result = ht.get(get); 3251 assertEmptyResult(result); 3252 3253 get = new Get(ROWS[ROWIDX]); 3254 get.addFamily(FAMILIES[4]); 3255 get.addColumn(FAMILIES[4], QUALIFIERS[2]); 3256 result = ht.get(get); 3257 assertEmptyResult(result); 3258 3259 get = new Get(ROWS[ROWIDX]); 3260 get.addFamily(FAMILIES[3]); 3261 get.addColumn(FAMILIES[4], QUALIFIERS[2]); 3262 get.addFamily(FAMILIES[5]); 3263 result = ht.get(get); 3264 assertEmptyResult(result); 3265 3266 get = new Get(ROWS[ROWIDX+1]); 3267 result = ht.get(get); 3268 assertEmptyResult(result); 3269 3270 } 3271 3272 private void scanVerifySingleEmpty(Table ht, 3273 byte [][] ROWS, int ROWIDX, 3274 byte [][] FAMILIES, int FAMILYIDX, 3275 byte [][] QUALIFIERS, int QUALIFIERIDX) 3276 throws Exception { 3277 3278 Scan scan = new Scan(ROWS[ROWIDX+1]); 3279 Result result = getSingleScanResult(ht, scan); 3280 assertNullResult(result); 3281 3282 scan = new Scan(ROWS[ROWIDX+1],ROWS[ROWIDX+2]); 3283 result = getSingleScanResult(ht, scan); 3284 assertNullResult(result); 3285 3286 scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX]); 3287 result = getSingleScanResult(ht, scan); 3288 assertNullResult(result); 3289 3290 scan = new Scan(); 3291 scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]); 3292 scan.addFamily(FAMILIES[FAMILYIDX-1]); 3293 result = getSingleScanResult(ht, scan); 3294 assertNullResult(result); 3295 3296 } 3297 3298 // 3299 // Verifiers 3300 // 3301 3302 private void assertKey(Cell key, byte [] row, byte [] family, 3303 byte [] qualifier, byte [] value) 3304 throws Exception { 3305 assertTrue("Expected row [" + Bytes.toString(row) + "] " + 3306 "Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]", 3307 equals(row, CellUtil.cloneRow(key))); 3308 assertTrue("Expected family [" + Bytes.toString(family) + "] " + 3309 "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]", 3310 equals(family, CellUtil.cloneFamily(key))); 3311 assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " + 3312 "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]", 3313 equals(qualifier, CellUtil.cloneQualifier(key))); 3314 assertTrue("Expected value [" + Bytes.toString(value) + "] " + 3315 "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]", 3316 equals(value, CellUtil.cloneValue(key))); 3317 } 3318 3319 static void assertIncrementKey(Cell key, byte [] row, byte [] family, 3320 byte [] qualifier, long value) 3321 throws Exception { 3322 assertTrue("Expected row [" + Bytes.toString(row) + "] " + 3323 "Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]", 3324 equals(row, CellUtil.cloneRow(key))); 3325 assertTrue("Expected family [" + Bytes.toString(family) + "] " + 3326 "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]", 3327 equals(family, CellUtil.cloneFamily(key))); 3328 assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " + 3329 "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]", 3330 equals(qualifier, CellUtil.cloneQualifier(key))); 3331 assertTrue("Expected value [" + value + "] " + 3332 "Got value [" + Bytes.toLong(CellUtil.cloneValue(key)) + "]", 3333 Bytes.toLong(CellUtil.cloneValue(key)) == value); 3334 } 3335 3336 private void assertNumKeys(Result result, int n) throws Exception { 3337 assertTrue("Expected " + n + " keys but got " + result.size(), 3338 result.size() == n); 3339 } 3340 3341 private void assertNResult(Result result, byte [] row, 3342 byte [][] families, byte [][] qualifiers, byte [][] values, 3343 int [][] idxs) 3344 throws Exception { 3345 assertTrue("Expected row [" + Bytes.toString(row) + "] " + 3346 "Got row [" + Bytes.toString(result.getRow()) +"]", 3347 equals(row, result.getRow())); 3348 assertTrue("Expected " + idxs.length + " keys but result contains " 3349 + result.size(), result.size() == idxs.length); 3350 3351 Cell [] keys = result.rawCells(); 3352 3353 for(int i=0;i<keys.length;i++) { 3354 byte [] family = families[idxs[i][0]]; 3355 byte [] qualifier = qualifiers[idxs[i][1]]; 3356 byte [] value = values[idxs[i][2]]; 3357 Cell key = keys[i]; 3358 3359 byte[] famb = CellUtil.cloneFamily(key); 3360 byte[] qualb = CellUtil.cloneQualifier(key); 3361 byte[] valb = CellUtil.cloneValue(key); 3362 assertTrue("(" + i + ") Expected family [" + Bytes.toString(family) 3363 + "] " + "Got family [" + Bytes.toString(famb) + "]", 3364 equals(family, famb)); 3365 assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier) 3366 + "] " + "Got qualifier [" + Bytes.toString(qualb) + "]", 3367 equals(qualifier, qualb)); 3368 assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] " 3369 + "Got value [" + Bytes.toString(valb) + "]", 3370 equals(value, valb)); 3371 } 3372 } 3373 3374 private void assertNResult(Result result, byte [] row, 3375 byte [] family, byte [] qualifier, long [] stamps, byte [][] values, 3376 int start, int end) 3377 throws IOException { 3378 assertTrue("Expected row [" + Bytes.toString(row) + "] " + 3379 "Got row [" + Bytes.toString(result.getRow()) +"]", 3380 equals(row, result.getRow())); 3381 int expectedResults = end - start + 1; 3382 assertEquals(expectedResults, result.size()); 3383 3384 Cell[] keys = result.rawCells(); 3385 3386 for (int i=0; i<keys.length; i++) { 3387 byte [] value = values[end-i]; 3388 long ts = stamps[end-i]; 3389 Cell key = keys[i]; 3390 3391 assertTrue("(" + i + ") Expected family [" + Bytes.toString(family) 3392 + "] " + "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]", 3393 CellUtil.matchingFamily(key, family)); 3394 assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier) 3395 + "] " + "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key))+ "]", 3396 CellUtil.matchingQualifier(key, qualifier)); 3397 assertTrue("Expected ts [" + ts + "] " + 3398 "Got ts [" + key.getTimestamp() + "]", ts == key.getTimestamp()); 3399 assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] " 3400 + "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]", 3401 CellUtil.matchingValue(key, value)); 3402 } 3403 } 3404 3405 /** 3406 * Validate that result contains two specified keys, exactly. 3407 * It is assumed key A sorts before key B. 3408 */ 3409 private void assertDoubleResult(Result result, byte [] row, 3410 byte [] familyA, byte [] qualifierA, byte [] valueA, 3411 byte [] familyB, byte [] qualifierB, byte [] valueB) 3412 throws Exception { 3413 assertTrue("Expected row [" + Bytes.toString(row) + "] " + 3414 "Got row [" + Bytes.toString(result.getRow()) +"]", 3415 equals(row, result.getRow())); 3416 assertTrue("Expected two keys but result contains " + result.size(), 3417 result.size() == 2); 3418 Cell [] kv = result.rawCells(); 3419 Cell kvA = kv[0]; 3420 assertTrue("(A) Expected family [" + Bytes.toString(familyA) + "] " + 3421 "Got family [" + Bytes.toString(CellUtil.cloneFamily(kvA)) + "]", 3422 equals(familyA, CellUtil.cloneFamily(kvA))); 3423 assertTrue("(A) Expected qualifier [" + Bytes.toString(qualifierA) + "] " + 3424 "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvA)) + "]", 3425 equals(qualifierA, CellUtil.cloneQualifier(kvA))); 3426 assertTrue("(A) Expected value [" + Bytes.toString(valueA) + "] " + 3427 "Got value [" + Bytes.toString(CellUtil.cloneValue(kvA)) + "]", 3428 equals(valueA, CellUtil.cloneValue(kvA))); 3429 Cell kvB = kv[1]; 3430 assertTrue("(B) Expected family [" + Bytes.toString(familyB) + "] " + 3431 "Got family [" + Bytes.toString(CellUtil.cloneFamily(kvB)) + "]", 3432 equals(familyB, CellUtil.cloneFamily(kvB))); 3433 assertTrue("(B) Expected qualifier [" + Bytes.toString(qualifierB) + "] " + 3434 "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvB)) + "]", 3435 equals(qualifierB, CellUtil.cloneQualifier(kvB))); 3436 assertTrue("(B) Expected value [" + Bytes.toString(valueB) + "] " + 3437 "Got value [" + Bytes.toString(CellUtil.cloneValue(kvB)) + "]", 3438 equals(valueB, CellUtil.cloneValue(kvB))); 3439 } 3440 3441 private void assertSingleResult(Result result, byte [] row, byte [] family, 3442 byte [] qualifier, byte [] value) 3443 throws Exception { 3444 assertTrue("Expected row [" + Bytes.toString(row) + "] " + 3445 "Got row [" + Bytes.toString(result.getRow()) +"]", 3446 equals(row, result.getRow())); 3447 assertTrue("Expected a single key but result contains " + result.size(), 3448 result.size() == 1); 3449 Cell kv = result.rawCells()[0]; 3450 assertTrue("Expected family [" + Bytes.toString(family) + "] " + 3451 "Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]", 3452 equals(family, CellUtil.cloneFamily(kv))); 3453 assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " + 3454 "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]", 3455 equals(qualifier, CellUtil.cloneQualifier(kv))); 3456 assertTrue("Expected value [" + Bytes.toString(value) + "] " + 3457 "Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]", 3458 equals(value, CellUtil.cloneValue(kv))); 3459 } 3460 3461 private void assertSingleResult(Result result, byte[] row, byte[] family, byte[] qualifier, 3462 long value) throws Exception { 3463 assertTrue( 3464 "Expected row [" + Bytes.toString(row) + "] " + "Got row [" + Bytes.toString(result.getRow()) 3465 + "]", equals(row, result.getRow())); 3466 assertTrue("Expected a single key but result contains " + result.size(), result.size() == 1); 3467 Cell kv = result.rawCells()[0]; 3468 assertTrue( 3469 "Expected family [" + Bytes.toString(family) + "] " + "Got family [" 3470 + Bytes.toString(CellUtil.cloneFamily(kv)) + "]", 3471 equals(family, CellUtil.cloneFamily(kv))); 3472 assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " + "Got qualifier [" 3473 + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]", 3474 equals(qualifier, CellUtil.cloneQualifier(kv))); 3475 assertTrue( 3476 "Expected value [" + value + "] " + "Got value [" + Bytes.toLong(CellUtil.cloneValue(kv)) 3477 + "]", value == Bytes.toLong(CellUtil.cloneValue(kv))); 3478 } 3479 3480 private void assertSingleResult(Result result, byte [] row, byte [] family, 3481 byte [] qualifier, long ts, byte [] value) 3482 throws Exception { 3483 assertTrue("Expected row [" + Bytes.toString(row) + "] " + 3484 "Got row [" + Bytes.toString(result.getRow()) +"]", 3485 equals(row, result.getRow())); 3486 assertTrue("Expected a single key but result contains " + result.size(), 3487 result.size() == 1); 3488 Cell kv = result.rawCells()[0]; 3489 assertTrue("Expected family [" + Bytes.toString(family) + "] " + 3490 "Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]", 3491 equals(family, CellUtil.cloneFamily(kv))); 3492 assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " + 3493 "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]", 3494 equals(qualifier, CellUtil.cloneQualifier(kv))); 3495 assertTrue("Expected ts [" + ts + "] " + 3496 "Got ts [" + kv.getTimestamp() + "]", ts == kv.getTimestamp()); 3497 assertTrue("Expected value [" + Bytes.toString(value) + "] " + 3498 "Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]", 3499 equals(value, CellUtil.cloneValue(kv))); 3500 } 3501 3502 private void assertEmptyResult(Result result) throws Exception { 3503 assertTrue("expected an empty result but result contains " + 3504 result.size() + " keys", result.isEmpty()); 3505 } 3506 3507 private void assertNullResult(Result result) throws Exception { 3508 assertTrue("expected null result but received a non-null result", 3509 result == null); 3510 } 3511 3512 // 3513 // Helpers 3514 // 3515 3516 private Result getSingleScanResult(Table ht, Scan scan) throws IOException { 3517 try (ResultScanner scanner = ht.getScanner(scan)) { 3518 Result result = scanner.next(); 3519 return result; 3520 } 3521 } 3522 3523 private byte [][] makeNAscii(byte [] base, int n) { 3524 if(n > 256) { 3525 return makeNBig(base, n); 3526 } 3527 byte [][] ret = new byte[n][]; 3528 for(int i=0;i<n;i++) { 3529 byte [] tail = Bytes.toBytes(Integer.toString(i)); 3530 ret[i] = Bytes.add(base, tail); 3531 } 3532 return ret; 3533 } 3534 3535 private byte [][] makeN(byte [] base, int n) { 3536 if (n > 256) { 3537 return makeNBig(base, n); 3538 } 3539 byte [][] ret = new byte[n][]; 3540 for(int i=0;i<n;i++) { 3541 ret[i] = Bytes.add(base, new byte[]{(byte)i}); 3542 } 3543 return ret; 3544 } 3545 3546 private byte [][] makeNBig(byte [] base, int n) { 3547 byte [][] ret = new byte[n][]; 3548 for(int i=0;i<n;i++) { 3549 int byteA = (i % 256); 3550 int byteB = (i >> 8); 3551 ret[i] = Bytes.add(base, new byte[]{(byte)byteB,(byte)byteA}); 3552 } 3553 return ret; 3554 } 3555 3556 private long [] makeStamps(int n) { 3557 long [] stamps = new long[n]; 3558 for (int i = 0; i < n; i++) { 3559 stamps[i] = i+1L; 3560 } 3561 return stamps; 3562 } 3563 3564 static boolean equals(byte [] left, byte [] right) { 3565 if (left == null && right == null) return true; 3566 if (left == null && right.length == 0) return true; 3567 if (right == null && left.length == 0) return true; 3568 return Bytes.equals(left, right); 3569 } 3570 3571 @Test 3572 public void testDuplicateVersions() throws Exception { 3573 final TableName tableName = TableName.valueOf(name.getMethodName()); 3574 3575 long [] STAMPS = makeStamps(20); 3576 byte [][] VALUES = makeNAscii(VALUE, 20); 3577 3578 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) { 3579 3580 // Insert 4 versions of same column 3581 Put put = new Put(ROW); 3582 put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 3583 put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 3584 put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 3585 put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 3586 ht.put(put); 3587 3588 // Verify we can get each one properly 3589 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 3590 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 3591 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 3592 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 3593 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 3594 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 3595 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 3596 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 3597 3598 // Verify we don't accidentally get others 3599 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 3600 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); 3601 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); 3602 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 3603 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); 3604 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); 3605 3606 // Ensure maxVersions in query is respected 3607 Get get = new Get(ROW); 3608 get.addColumn(FAMILY, QUALIFIER); 3609 get.setMaxVersions(2); 3610 Result result = ht.get(get); 3611 assertNResult(result, ROW, FAMILY, QUALIFIER, 3612 new long [] {STAMPS[4], STAMPS[5]}, 3613 new byte[][] {VALUES[4], VALUES[5]}, 3614 0, 1); 3615 3616 Scan scan = new Scan(ROW); 3617 scan.addColumn(FAMILY, QUALIFIER); 3618 scan.setMaxVersions(2); 3619 result = getSingleScanResult(ht, scan); 3620 assertNResult(result, ROW, FAMILY, QUALIFIER, 3621 new long[]{STAMPS[4], STAMPS[5]}, 3622 new byte[][]{VALUES[4], VALUES[5]}, 3623 0, 1); 3624 3625 // Flush and redo 3626 3627 TEST_UTIL.flush(); 3628 3629 // Verify we can get each one properly 3630 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 3631 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 3632 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 3633 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 3634 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 3635 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 3636 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 3637 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 3638 3639 // Verify we don't accidentally get others 3640 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 3641 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); 3642 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); 3643 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 3644 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); 3645 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); 3646 3647 // Ensure maxVersions in query is respected 3648 get = new Get(ROW); 3649 get.addColumn(FAMILY, QUALIFIER); 3650 get.setMaxVersions(2); 3651 result = ht.get(get); 3652 assertNResult(result, ROW, FAMILY, QUALIFIER, 3653 new long [] {STAMPS[4], STAMPS[5]}, 3654 new byte[][] {VALUES[4], VALUES[5]}, 3655 0, 1); 3656 3657 scan = new Scan(ROW); 3658 scan.addColumn(FAMILY, QUALIFIER); 3659 scan.setMaxVersions(2); 3660 result = getSingleScanResult(ht, scan); 3661 assertNResult(result, ROW, FAMILY, QUALIFIER, 3662 new long[]{STAMPS[4], STAMPS[5]}, 3663 new byte[][]{VALUES[4], VALUES[5]}, 3664 0, 1); 3665 3666 3667 // Add some memstore and retest 3668 3669 // Insert 4 more versions of same column and a dupe 3670 put = new Put(ROW); 3671 put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); 3672 put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]); 3673 put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]); 3674 put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]); 3675 put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]); 3676 ht.put(put); 3677 3678 // Ensure maxVersions in query is respected 3679 get = new Get(ROW); 3680 get.addColumn(FAMILY, QUALIFIER); 3681 get.setMaxVersions(7); 3682 result = ht.get(get); 3683 assertNResult(result, ROW, FAMILY, QUALIFIER, 3684 new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]}, 3685 new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], 3686 VALUES[8]}, 3687 0, 6); 3688 3689 scan = new Scan(ROW); 3690 scan.addColumn(FAMILY, QUALIFIER); 3691 scan.setMaxVersions(7); 3692 result = getSingleScanResult(ht, scan); 3693 assertNResult(result, ROW, FAMILY, QUALIFIER, 3694 new long[]{STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]}, 3695 new byte[][]{VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]}, 3696 0, 6); 3697 3698 get = new Get(ROW); 3699 get.setMaxVersions(7); 3700 result = ht.get(get); 3701 assertNResult(result, ROW, FAMILY, QUALIFIER, 3702 new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]}, 3703 new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], 3704 VALUES[8]}, 3705 0, 6); 3706 3707 scan = new Scan(ROW); 3708 scan.setMaxVersions(7); 3709 result = getSingleScanResult(ht, scan); 3710 assertNResult(result, ROW, FAMILY, QUALIFIER, 3711 new long[]{STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]}, 3712 new byte[][]{VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]}, 3713 0, 6); 3714 3715 // Verify we can get each one properly 3716 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 3717 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 3718 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]); 3719 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]); 3720 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 3721 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 3722 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]); 3723 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]); 3724 3725 // Verify we don't accidentally get others 3726 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 3727 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]); 3728 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 3729 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]); 3730 3731 // Ensure maxVersions of table is respected 3732 3733 TEST_UTIL.flush(); 3734 3735 // Insert 4 more versions of same column and a dupe 3736 put = new Put(ROW); 3737 put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]); 3738 put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]); 3739 put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]); 3740 put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]); 3741 ht.put(put); 3742 3743 get = new Get(ROW); 3744 get.addColumn(FAMILY, QUALIFIER); 3745 get.setMaxVersions(Integer.MAX_VALUE); 3746 result = ht.get(get); 3747 assertNResult(result, ROW, FAMILY, QUALIFIER, 3748 new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], 3749 STAMPS[11], STAMPS[13], STAMPS[15]}, 3750 new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], 3751 VALUES[9], VALUES[11], VALUES[13], VALUES[15]}, 3752 0, 9); 3753 3754 scan = new Scan(ROW); 3755 scan.addColumn(FAMILY, QUALIFIER); 3756 scan.setMaxVersions(Integer.MAX_VALUE); 3757 result = getSingleScanResult(ht, scan); 3758 assertNResult(result, ROW, FAMILY, QUALIFIER, 3759 new long[]{STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], 3760 STAMPS[11], STAMPS[13], STAMPS[15]}, 3761 new byte[][]{VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], 3762 VALUES[11], VALUES[13], VALUES[15]},0, 9); 3763 3764 // Delete a version in the memstore and a version in a storefile 3765 Delete delete = new Delete(ROW); 3766 delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]); 3767 delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]); 3768 ht.delete(delete); 3769 3770 // Test that it's gone 3771 get = new Get(ROW); 3772 get.addColumn(FAMILY, QUALIFIER); 3773 get.setMaxVersions(Integer.MAX_VALUE); 3774 result = ht.get(get); 3775 assertNResult(result, ROW, FAMILY, QUALIFIER, 3776 new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], 3777 STAMPS[9], STAMPS[13], STAMPS[15]}, 3778 new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], 3779 VALUES[8], VALUES[9], VALUES[13], VALUES[15]}, 3780 0, 9); 3781 3782 scan = new Scan(ROW); 3783 scan.addColumn(FAMILY, QUALIFIER); 3784 scan.setMaxVersions(Integer.MAX_VALUE); 3785 result = getSingleScanResult(ht, scan); 3786 assertNResult(result, ROW, FAMILY, QUALIFIER, 3787 new long[]{STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], 3788 STAMPS[9], STAMPS[13], STAMPS[15]}, 3789 new byte[][]{VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], 3790 VALUES[9], VALUES[13], VALUES[15]},0,9); 3791 } 3792 } 3793 3794 @Test 3795 public void testUpdates() throws Exception { 3796 final TableName tableName = TableName.valueOf(name.getMethodName()); 3797 try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10)) { 3798 3799 // Write a column with values at timestamp 1, 2 and 3 3800 byte[] row = Bytes.toBytes("row1"); 3801 byte[] qualifier = Bytes.toBytes("myCol"); 3802 Put put = new Put(row); 3803 put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA")); 3804 hTable.put(put); 3805 3806 put = new Put(row); 3807 put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB")); 3808 hTable.put(put); 3809 3810 put = new Put(row); 3811 put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE")); 3812 hTable.put(put); 3813 3814 Get get = new Get(row); 3815 get.addColumn(FAMILY, qualifier); 3816 get.setMaxVersions(); 3817 3818 // Check that the column indeed has the right values at timestamps 1 and 3819 // 2 3820 Result result = hTable.get(get); 3821 NavigableMap<Long, byte[]> navigableMap = 3822 result.getMap().get(FAMILY).get(qualifier); 3823 assertEquals("AAA", Bytes.toString(navigableMap.get(1L))); 3824 assertEquals("BBB", Bytes.toString(navigableMap.get(2L))); 3825 3826 // Update the value at timestamp 1 3827 put = new Put(row); 3828 put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC")); 3829 hTable.put(put); 3830 3831 // Update the value at timestamp 2 3832 put = new Put(row); 3833 put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD")); 3834 hTable.put(put); 3835 3836 // Check that the values at timestamp 2 and 1 got updated 3837 result = hTable.get(get); 3838 navigableMap = result.getMap().get(FAMILY).get(qualifier); 3839 assertEquals("CCC", Bytes.toString(navigableMap.get(1L))); 3840 assertEquals("DDD", Bytes.toString(navigableMap.get(2L))); 3841 } 3842 } 3843 3844 @Test 3845 public void testUpdatesWithMajorCompaction() throws Exception { 3846 final TableName tableName = TableName.valueOf(name.getMethodName()); 3847 try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10); 3848 Admin admin = TEST_UTIL.getAdmin()) { 3849 3850 // Write a column with values at timestamp 1, 2 and 3 3851 byte[] row = Bytes.toBytes("row2"); 3852 byte[] qualifier = Bytes.toBytes("myCol"); 3853 Put put = new Put(row); 3854 put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA")); 3855 hTable.put(put); 3856 3857 put = new Put(row); 3858 put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB")); 3859 hTable.put(put); 3860 3861 put = new Put(row); 3862 put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE")); 3863 hTable.put(put); 3864 3865 Get get = new Get(row); 3866 get.addColumn(FAMILY, qualifier); 3867 get.setMaxVersions(); 3868 3869 // Check that the column indeed has the right values at timestamps 1 and 3870 // 2 3871 Result result = hTable.get(get); 3872 NavigableMap<Long, byte[]> navigableMap = 3873 result.getMap().get(FAMILY).get(qualifier); 3874 assertEquals("AAA", Bytes.toString(navigableMap.get(1L))); 3875 assertEquals("BBB", Bytes.toString(navigableMap.get(2L))); 3876 3877 // Trigger a major compaction 3878 admin.flush(tableName); 3879 admin.majorCompact(tableName); 3880 Thread.sleep(6000); 3881 3882 // Update the value at timestamp 1 3883 put = new Put(row); 3884 put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC")); 3885 hTable.put(put); 3886 3887 // Update the value at timestamp 2 3888 put = new Put(row); 3889 put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD")); 3890 hTable.put(put); 3891 3892 // Trigger a major compaction 3893 admin.flush(tableName); 3894 admin.majorCompact(tableName); 3895 Thread.sleep(6000); 3896 3897 // Check that the values at timestamp 2 and 1 got updated 3898 result = hTable.get(get); 3899 navigableMap = result.getMap().get(FAMILY).get(qualifier); 3900 assertEquals("CCC", Bytes.toString(navigableMap.get(1L))); 3901 assertEquals("DDD", Bytes.toString(navigableMap.get(2L))); 3902 } 3903 } 3904 3905 @Test 3906 public void testMajorCompactionBetweenTwoUpdates() throws Exception { 3907 final TableName tableName = TableName.valueOf(name.getMethodName()); 3908 try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10); 3909 Admin admin = TEST_UTIL.getAdmin()) { 3910 3911 // Write a column with values at timestamp 1, 2 and 3 3912 byte[] row = Bytes.toBytes("row3"); 3913 byte[] qualifier = Bytes.toBytes("myCol"); 3914 Put put = new Put(row); 3915 put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA")); 3916 hTable.put(put); 3917 3918 put = new Put(row); 3919 put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB")); 3920 hTable.put(put); 3921 3922 put = new Put(row); 3923 put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE")); 3924 hTable.put(put); 3925 3926 Get get = new Get(row); 3927 get.addColumn(FAMILY, qualifier); 3928 get.setMaxVersions(); 3929 3930 // Check that the column indeed has the right values at timestamps 1 and 3931 // 2 3932 Result result = hTable.get(get); 3933 NavigableMap<Long, byte[]> navigableMap = 3934 result.getMap().get(FAMILY).get(qualifier); 3935 assertEquals("AAA", Bytes.toString(navigableMap.get(1L))); 3936 assertEquals("BBB", Bytes.toString(navigableMap.get(2L))); 3937 3938 // Trigger a major compaction 3939 admin.flush(tableName); 3940 admin.majorCompact(tableName); 3941 Thread.sleep(6000); 3942 3943 // Update the value at timestamp 1 3944 put = new Put(row); 3945 put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC")); 3946 hTable.put(put); 3947 3948 // Trigger a major compaction 3949 admin.flush(tableName); 3950 admin.majorCompact(tableName); 3951 Thread.sleep(6000); 3952 3953 // Update the value at timestamp 2 3954 put = new Put(row); 3955 put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD")); 3956 hTable.put(put); 3957 3958 // Trigger a major compaction 3959 admin.flush(tableName); 3960 admin.majorCompact(tableName); 3961 Thread.sleep(6000); 3962 3963 // Check that the values at timestamp 2 and 1 got updated 3964 result = hTable.get(get); 3965 navigableMap = result.getMap().get(FAMILY).get(qualifier); 3966 3967 assertEquals("CCC", Bytes.toString(navigableMap.get(1L))); 3968 assertEquals("DDD", Bytes.toString(navigableMap.get(2L))); 3969 } 3970 } 3971 3972 @Test 3973 public void testGet_EmptyTable() throws IOException { 3974 try (Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY)) { 3975 Get get = new Get(ROW); 3976 get.addFamily(FAMILY); 3977 Result r = table.get(get); 3978 assertTrue(r.isEmpty()); 3979 } 3980 } 3981 3982 @Test 3983 public void testGet_NullQualifier() throws IOException { 3984 try (Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY)) { 3985 Put put = new Put(ROW); 3986 put.addColumn(FAMILY, QUALIFIER, VALUE); 3987 table.put(put); 3988 3989 put = new Put(ROW); 3990 put.addColumn(FAMILY, null, VALUE); 3991 table.put(put); 3992 LOG.info("Row put"); 3993 3994 Get get = new Get(ROW); 3995 get.addColumn(FAMILY, null); 3996 Result r = table.get(get); 3997 assertEquals(1, r.size()); 3998 3999 get = new Get(ROW); 4000 get.addFamily(FAMILY); 4001 r = table.get(get); 4002 assertEquals(2, r.size()); 4003 } 4004 } 4005 4006 @Test 4007 public void testGet_NonExistentRow() throws IOException { 4008 try (Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY)) { 4009 Put put = new Put(ROW); 4010 put.addColumn(FAMILY, QUALIFIER, VALUE); 4011 table.put(put); 4012 LOG.info("Row put"); 4013 4014 Get get = new Get(ROW); 4015 get.addFamily(FAMILY); 4016 Result r = table.get(get); 4017 assertFalse(r.isEmpty()); 4018 System.out.println("Row retrieved successfully"); 4019 4020 byte[] missingrow = Bytes.toBytes("missingrow"); 4021 get = new Get(missingrow); 4022 get.addFamily(FAMILY); 4023 r = table.get(get); 4024 assertTrue(r.isEmpty()); 4025 LOG.info("Row missing as it should be"); 4026 } 4027 } 4028 4029 @Test 4030 public void testPut() throws IOException { 4031 final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents"); 4032 final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam"); 4033 final byte [] row1 = Bytes.toBytes("row1"); 4034 final byte [] row2 = Bytes.toBytes("row2"); 4035 final byte [] value = Bytes.toBytes("abcd"); 4036 try (Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), 4037 new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) { 4038 Put put = new Put(row1); 4039 put.addColumn(CONTENTS_FAMILY, null, value); 4040 table.put(put); 4041 4042 put = new Put(row2); 4043 put.addColumn(CONTENTS_FAMILY, null, value); 4044 4045 assertEquals(1, put.size()); 4046 assertEquals(1, put.getFamilyCellMap().get(CONTENTS_FAMILY).size()); 4047 4048 // KeyValue v1 expectation. Cast for now until we go all Cell all the time. TODO 4049 KeyValue kv = (KeyValue) put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0); 4050 4051 assertTrue(Bytes.equals(CellUtil.cloneFamily(kv), CONTENTS_FAMILY)); 4052 // will it return null or an empty byte array? 4053 assertTrue(Bytes.equals(CellUtil.cloneQualifier(kv), new byte[0])); 4054 4055 assertTrue(Bytes.equals(CellUtil.cloneValue(kv), value)); 4056 4057 table.put(put); 4058 4059 Scan scan = new Scan(); 4060 scan.addColumn(CONTENTS_FAMILY, null); 4061 try (ResultScanner scanner = table.getScanner(scan)) { 4062 for (Result r : scanner) { 4063 for (Cell key : r.rawCells()) { 4064 System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString()); 4065 } 4066 } 4067 } 4068 } 4069 } 4070 4071 @Test 4072 public void testPutNoCF() throws IOException { 4073 final byte[] BAD_FAM = Bytes.toBytes("BAD_CF"); 4074 final byte[] VAL = Bytes.toBytes(100); 4075 try (Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY)) { 4076 boolean caughtNSCFE = false; 4077 4078 try { 4079 Put p = new Put(ROW); 4080 p.addColumn(BAD_FAM, QUALIFIER, VAL); 4081 table.put(p); 4082 } catch (Exception e) { 4083 caughtNSCFE = e instanceof NoSuchColumnFamilyException; 4084 } 4085 assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE); 4086 } 4087 } 4088 4089 @Test 4090 public void testRowsPut() throws IOException { 4091 final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents"); 4092 final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam"); 4093 final int NB_BATCH_ROWS = 10; 4094 final byte[] value = Bytes.toBytes("abcd"); 4095 try (Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), 4096 new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY })) { 4097 ArrayList<Put> rowsUpdate = new ArrayList<Put>(); 4098 for (int i = 0; i < NB_BATCH_ROWS; i++) { 4099 byte[] row = Bytes.toBytes("row" + i); 4100 Put put = new Put(row); 4101 put.setDurability(Durability.SKIP_WAL); 4102 put.addColumn(CONTENTS_FAMILY, null, value); 4103 rowsUpdate.add(put); 4104 } 4105 table.put(rowsUpdate); 4106 Scan scan = new Scan(); 4107 scan.addFamily(CONTENTS_FAMILY); 4108 try (ResultScanner scanner = table.getScanner(scan)) { 4109 int nbRows = 0; 4110 for (@SuppressWarnings("unused") 4111 Result row : scanner) { 4112 nbRows++; 4113 } 4114 assertEquals(NB_BATCH_ROWS, nbRows); 4115 } 4116 } 4117 } 4118 4119 @Test 4120 public void testRowsPutBufferedManyManyFlushes() throws IOException { 4121 final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents"); 4122 final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam"); 4123 final byte[] value = Bytes.toBytes("abcd"); 4124 final int NB_BATCH_ROWS = 10; 4125 try (Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), 4126 new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) { 4127 ArrayList<Put> rowsUpdate = new ArrayList<Put>(); 4128 for (int i = 0; i < NB_BATCH_ROWS * 10; i++) { 4129 byte[] row = Bytes.toBytes("row" + i); 4130 Put put = new Put(row); 4131 put.setDurability(Durability.SKIP_WAL); 4132 put.addColumn(CONTENTS_FAMILY, null, value); 4133 rowsUpdate.add(put); 4134 } 4135 table.put(rowsUpdate); 4136 4137 Scan scan = new Scan(); 4138 scan.addFamily(CONTENTS_FAMILY); 4139 try (ResultScanner scanner = table.getScanner(scan)) { 4140 int nbRows = 0; 4141 for (@SuppressWarnings("unused") 4142 Result row : scanner) { 4143 nbRows++; 4144 } 4145 assertEquals(NB_BATCH_ROWS * 10, nbRows); 4146 } 4147 } 4148 } 4149 4150 @Test 4151 public void testAddKeyValue() throws IOException { 4152 final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents"); 4153 final byte[] value = Bytes.toBytes("abcd"); 4154 final byte[] row1 = Bytes.toBytes("row1"); 4155 final byte[] row2 = Bytes.toBytes("row2"); 4156 byte[] qualifier = Bytes.toBytes("qf1"); 4157 Put put = new Put(row1); 4158 4159 // Adding KeyValue with the same row 4160 KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value); 4161 boolean ok = true; 4162 try { 4163 put.add(kv); 4164 } catch (IOException e) { 4165 ok = false; 4166 } 4167 assertEquals(true, ok); 4168 4169 // Adding KeyValue with the different row 4170 kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value); 4171 ok = false; 4172 try { 4173 put.add(kv); 4174 } catch (IOException e) { 4175 ok = true; 4176 } 4177 assertEquals(true, ok); 4178 } 4179 4180 /** 4181 * test for HBASE-737 4182 */ 4183 @Test 4184 public void testHBase737 () throws IOException { 4185 final byte [] FAM1 = Bytes.toBytes("fam1"); 4186 final byte [] FAM2 = Bytes.toBytes("fam2"); 4187 // Open table 4188 try (Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), 4189 new byte [][] {FAM1, FAM2})) { 4190 // Insert some values 4191 Put put = new Put(ROW); 4192 put.addColumn(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg")); 4193 table.put(put); 4194 try { 4195 Thread.sleep(1000); 4196 } catch (InterruptedException i) { 4197 //ignore 4198 } 4199 4200 put = new Put(ROW); 4201 put.addColumn(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456")); 4202 table.put(put); 4203 4204 try { 4205 Thread.sleep(1000); 4206 } catch (InterruptedException i) { 4207 //ignore 4208 } 4209 4210 put = new Put(ROW); 4211 put.addColumn(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop")); 4212 table.put(put); 4213 4214 long[] times = new long[3]; 4215 4216 // First scan the memstore 4217 4218 Scan scan = new Scan(); 4219 scan.addFamily(FAM1); 4220 scan.addFamily(FAM2); 4221 try (ResultScanner s = table.getScanner(scan)) { 4222 int index = 0; 4223 Result r = null; 4224 while ((r = s.next()) != null) { 4225 for (Cell key : r.rawCells()) { 4226 times[index++] = key.getTimestamp(); 4227 } 4228 } 4229 } 4230 for (int i = 0; i < times.length - 1; i++) { 4231 for (int j = i + 1; j < times.length; j++) { 4232 assertTrue(times[j] > times[i]); 4233 } 4234 } 4235 4236 // Flush data to disk and try again 4237 TEST_UTIL.flush(); 4238 4239 // Reset times 4240 for (int i = 0; i < times.length; i++) { 4241 times[i] = 0; 4242 } 4243 4244 try { 4245 Thread.sleep(1000); 4246 } catch (InterruptedException i) { 4247 //ignore 4248 } 4249 scan = new Scan(); 4250 scan.addFamily(FAM1); 4251 scan.addFamily(FAM2); 4252 try (ResultScanner s = table.getScanner(scan)) { 4253 int index = 0; 4254 Result r = null; 4255 while ((r = s.next()) != null) { 4256 for (Cell key : r.rawCells()) { 4257 times[index++] = key.getTimestamp(); 4258 } 4259 } 4260 for (int i = 0; i < times.length - 1; i++) { 4261 for (int j = i + 1; j < times.length; j++) { 4262 assertTrue(times[j] > times[i]); 4263 } 4264 } 4265 } 4266 } 4267 } 4268 4269 @Test 4270 public void testListTables() throws IOException, InterruptedException { 4271 final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); 4272 final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); 4273 final TableName tableName3 = TableName.valueOf(name.getMethodName() + "3"); 4274 TableName [] tables = new TableName[] { tableName1, tableName2, tableName3 }; 4275 for (int i = 0; i < tables.length; i++) { 4276 TEST_UTIL.createTable(tables[i], FAMILY); 4277 } 4278 try (Admin admin = TEST_UTIL.getAdmin()) { 4279 List<TableDescriptor> ts = admin.listTableDescriptors(); 4280 HashSet<TableDescriptor> result = new HashSet<>(ts); 4281 int size = result.size(); 4282 assertTrue(size >= tables.length); 4283 for (int i = 0; i < tables.length && i < size; i++) { 4284 boolean found = false; 4285 for (int j = 0; j < ts.size(); j++) { 4286 if (ts.get(j).getTableName().equals(tables[i])) { 4287 found = true; 4288 break; 4289 } 4290 } 4291 assertTrue("Not found: " + tables[i], found); 4292 } 4293 } 4294 } 4295 4296 /** 4297 * simple test that just executes parts of the client 4298 * API that accept a pre-created Connection instance 4299 */ 4300 @Test 4301 public void testUnmanagedHConnection() throws IOException { 4302 final TableName tableName = TableName.valueOf(name.getMethodName()); 4303 TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY); 4304 try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); 4305 Table t = conn.getTable(tableName); 4306 Admin admin = conn.getAdmin()) { 4307 assertTrue(admin.tableExists(tableName)); 4308 assertTrue(t.get(new Get(ROW)).isEmpty()); 4309 } 4310 } 4311 4312 /** 4313 * test of that unmanaged HConnections are able to reconnect 4314 * properly (see HBASE-5058) 4315 */ 4316 @Test 4317 public void testUnmanagedHConnectionReconnect() throws Exception { 4318 final TableName tableName = TableName.valueOf(name.getMethodName()); 4319 TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY); 4320 try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) { 4321 try (Table t = conn.getTable(tableName); 4322 Admin admin = conn.getAdmin()) { 4323 assertTrue(admin.tableExists(tableName)); 4324 assertTrue(t.get(new Get(ROW)).isEmpty()); 4325 } 4326 4327 // stop the master 4328 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 4329 cluster.stopMaster(0, false); 4330 cluster.waitOnMaster(0); 4331 4332 // start up a new master 4333 cluster.startMaster(); 4334 assertTrue(cluster.waitForActiveAndReadyMaster()); 4335 4336 // test that the same unmanaged connection works with a new 4337 // Admin and can connect to the new master; 4338 boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration()); 4339 try (Admin admin = conn.getAdmin()) { 4340 assertTrue(admin.tableExists(tableName)); 4341 assertTrue(admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)) 4342 .getLiveServerMetrics().size() == SLAVES + (tablesOnMaster ? 1 : 0)); 4343 } 4344 } 4345 } 4346 4347 @Test 4348 public void testMiscHTableStuff() throws IOException { 4349 final TableName tableAname = TableName.valueOf(name.getMethodName() + "A"); 4350 final TableName tableBname = TableName.valueOf(name.getMethodName() + "B"); 4351 final byte[] attrName = Bytes.toBytes("TESTATTR"); 4352 final byte[] attrValue = Bytes.toBytes("somevalue"); 4353 byte[] value = Bytes.toBytes("value"); 4354 4355 try (Table a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY); 4356 Table b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY)) { 4357 Put put = new Put(ROW); 4358 put.addColumn(HConstants.CATALOG_FAMILY, null, value); 4359 a.put(put); 4360 4361 // open a new connection to A and a connection to b 4362 try (Table newA = TEST_UTIL.getConnection().getTable(tableAname)) { 4363 4364 // copy data from A to B 4365 Scan scan = new Scan(); 4366 scan.addFamily(HConstants.CATALOG_FAMILY); 4367 try (ResultScanner s = newA.getScanner(scan)) { 4368 for (Result r : s) { 4369 put = new Put(r.getRow()); 4370 put.setDurability(Durability.SKIP_WAL); 4371 for (Cell kv : r.rawCells()) { 4372 put.add(kv); 4373 } 4374 b.put(put); 4375 } 4376 } 4377 } 4378 4379 // Opening a new connection to A will cause the tables to be reloaded 4380 try (Table anotherA = TEST_UTIL.getConnection().getTable(tableAname)) { 4381 Get get = new Get(ROW); 4382 get.addFamily(HConstants.CATALOG_FAMILY); 4383 anotherA.get(get); 4384 } 4385 4386 // We can still access A through newA because it has the table information 4387 // cached. And if it needs to recalibrate, that will cause the information 4388 // to be reloaded. 4389 4390 // Test user metadata 4391 try (Admin admin = TEST_UTIL.getAdmin()) { 4392 // make a modifiable descriptor 4393 HTableDescriptor desc = new HTableDescriptor(a.getTableDescriptor()); 4394 // offline the table 4395 admin.disableTable(tableAname); 4396 // add a user attribute to HTD 4397 desc.setValue(attrName, attrValue); 4398 // add a user attribute to HCD 4399 for (HColumnDescriptor c : desc.getFamilies()) { 4400 c.setValue(attrName, attrValue); 4401 } 4402 // update metadata for all regions of this table 4403 admin.modifyTable(desc); 4404 // enable the table 4405 admin.enableTable(tableAname); 4406 } 4407 4408 // Test that attribute changes were applied 4409 HTableDescriptor desc = a.getTableDescriptor(); 4410 assertEquals("wrong table descriptor returned", desc.getTableName(), tableAname); 4411 // check HTD attribute 4412 value = desc.getValue(attrName); 4413 assertFalse("missing HTD attribute value", value == null); 4414 assertFalse("HTD attribute value is incorrect", 4415 Bytes.compareTo(value, attrValue) != 0); 4416 // check HCD attribute 4417 for (HColumnDescriptor c : desc.getFamilies()) { 4418 value = c.getValue(attrName); 4419 assertFalse("missing HCD attribute value", value == null); 4420 assertFalse("HCD attribute value is incorrect", 4421 Bytes.compareTo(value, attrValue) != 0); 4422 } 4423 } 4424 } 4425 4426 @Test 4427 public void testGetClosestRowBefore() throws IOException, InterruptedException { 4428 final TableName tableName = TableName.valueOf(name.getMethodName()); 4429 final byte[] firstRow = Bytes.toBytes("row111"); 4430 final byte[] secondRow = Bytes.toBytes("row222"); 4431 final byte[] thirdRow = Bytes.toBytes("row333"); 4432 final byte[] forthRow = Bytes.toBytes("row444"); 4433 final byte[] beforeFirstRow = Bytes.toBytes("row"); 4434 final byte[] beforeSecondRow = Bytes.toBytes("row22"); 4435 final byte[] beforeThirdRow = Bytes.toBytes("row33"); 4436 final byte[] beforeForthRow = Bytes.toBytes("row44"); 4437 4438 try (Table table = 4439 TEST_UTIL.createTable(tableName, 4440 new byte[][] { HConstants.CATALOG_FAMILY, Bytes.toBytes("info2") }, 1, 1024); 4441 RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { 4442 4443 // set block size to 64 to making 2 kvs into one block, bypassing the walkForwardInSingleRow 4444 // in Store.rowAtOrBeforeFromStoreFile 4445 String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName(); 4446 HRegion region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName); 4447 Put put1 = new Put(firstRow); 4448 Put put2 = new Put(secondRow); 4449 Put put3 = new Put(thirdRow); 4450 Put put4 = new Put(forthRow); 4451 byte[] one = new byte[] { 1 }; 4452 byte[] two = new byte[] { 2 }; 4453 byte[] three = new byte[] { 3 }; 4454 byte[] four = new byte[] { 4 }; 4455 4456 put1.addColumn(HConstants.CATALOG_FAMILY, null, one); 4457 put2.addColumn(HConstants.CATALOG_FAMILY, null, two); 4458 put3.addColumn(HConstants.CATALOG_FAMILY, null, three); 4459 put4.addColumn(HConstants.CATALOG_FAMILY, null, four); 4460 table.put(put1); 4461 table.put(put2); 4462 table.put(put3); 4463 table.put(put4); 4464 region.flush(true); 4465 4466 Result result; 4467 4468 // Test before first that null is returned 4469 result = getReverseScanResult(table, beforeFirstRow, 4470 HConstants.CATALOG_FAMILY); 4471 assertNull(result); 4472 4473 // Test at first that first is returned 4474 result = getReverseScanResult(table, firstRow, HConstants.CATALOG_FAMILY); 4475 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null)); 4476 assertTrue(Bytes.equals(result.getRow(), firstRow)); 4477 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one)); 4478 4479 // Test in between first and second that first is returned 4480 result = getReverseScanResult(table, beforeSecondRow, HConstants.CATALOG_FAMILY); 4481 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null)); 4482 assertTrue(Bytes.equals(result.getRow(), firstRow)); 4483 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one)); 4484 4485 // Test at second make sure second is returned 4486 result = getReverseScanResult(table, secondRow, HConstants.CATALOG_FAMILY); 4487 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null)); 4488 assertTrue(Bytes.equals(result.getRow(), secondRow)); 4489 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two)); 4490 4491 // Test in second and third, make sure second is returned 4492 result = getReverseScanResult(table, beforeThirdRow, HConstants.CATALOG_FAMILY); 4493 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null)); 4494 assertTrue(Bytes.equals(result.getRow(), secondRow)); 4495 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two)); 4496 4497 // Test at third make sure third is returned 4498 result = getReverseScanResult(table, thirdRow, HConstants.CATALOG_FAMILY); 4499 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null)); 4500 assertTrue(Bytes.equals(result.getRow(), thirdRow)); 4501 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three)); 4502 4503 // Test in third and forth, make sure third is returned 4504 result = getReverseScanResult(table, beforeForthRow, HConstants.CATALOG_FAMILY); 4505 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null)); 4506 assertTrue(Bytes.equals(result.getRow(), thirdRow)); 4507 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three)); 4508 4509 // Test at forth make sure forth is returned 4510 result = getReverseScanResult(table, forthRow, HConstants.CATALOG_FAMILY); 4511 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null)); 4512 assertTrue(Bytes.equals(result.getRow(), forthRow)); 4513 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four)); 4514 4515 // Test after forth make sure forth is returned 4516 result = getReverseScanResult(table, Bytes.add(forthRow, one), HConstants.CATALOG_FAMILY); 4517 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null)); 4518 assertTrue(Bytes.equals(result.getRow(), forthRow)); 4519 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four)); 4520 } 4521 } 4522 4523 private Result getReverseScanResult(Table table, byte[] row, byte[] fam) throws IOException { 4524 Scan scan = new Scan(row); 4525 scan.setSmall(true); 4526 scan.setReversed(true); 4527 scan.setCaching(1); 4528 scan.addFamily(fam); 4529 try (ResultScanner scanner = table.getScanner(scan)) { 4530 return scanner.next(); 4531 } 4532 } 4533 4534 /** 4535 * For HBASE-2156 4536 */ 4537 @Test 4538 public void testScanVariableReuse() throws Exception { 4539 Scan scan = new Scan(); 4540 scan.addFamily(FAMILY); 4541 scan.addColumn(FAMILY, ROW); 4542 4543 assertTrue(scan.getFamilyMap().get(FAMILY).size() == 1); 4544 4545 scan = new Scan(); 4546 scan.addFamily(FAMILY); 4547 4548 assertTrue(scan.getFamilyMap().get(FAMILY) == null); 4549 assertTrue(scan.getFamilyMap().containsKey(FAMILY)); 4550 } 4551 4552 @Test 4553 public void testMultiRowMutation() throws Exception { 4554 LOG.info("Starting testMultiRowMutation"); 4555 final TableName tableName = TableName.valueOf(name.getMethodName()); 4556 final byte [] ROW1 = Bytes.toBytes("testRow1"); 4557 4558 try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) { 4559 Put p = new Put(ROW); 4560 p.addColumn(FAMILY, QUALIFIER, VALUE); 4561 MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, p); 4562 4563 p = new Put(ROW1); 4564 p.addColumn(FAMILY, QUALIFIER, VALUE); 4565 MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, p); 4566 4567 MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder(); 4568 mrmBuilder.addMutationRequest(m1); 4569 mrmBuilder.addMutationRequest(m2); 4570 MutateRowsRequest mrm = mrmBuilder.build(); 4571 CoprocessorRpcChannel channel = t.coprocessorService(ROW); 4572 MultiRowMutationService.BlockingInterface service = 4573 MultiRowMutationService.newBlockingStub(channel); 4574 service.mutateRows(null, mrm); 4575 Get g = new Get(ROW); 4576 Result r = t.get(g); 4577 assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER))); 4578 g = new Get(ROW1); 4579 r = t.get(g); 4580 assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER))); 4581 } 4582 } 4583 4584 @Test 4585 public void testRowMutation() throws Exception { 4586 LOG.info("Starting testRowMutation"); 4587 final TableName tableName = TableName.valueOf(name.getMethodName()); 4588 try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) { 4589 byte[][] QUALIFIERS = new byte[][]{ 4590 Bytes.toBytes("a"), Bytes.toBytes("b") 4591 }; 4592 RowMutations arm = new RowMutations(ROW); 4593 Put p = new Put(ROW); 4594 p.addColumn(FAMILY, QUALIFIERS[0], VALUE); 4595 arm.add(p); 4596 t.mutateRow(arm); 4597 4598 Get g = new Get(ROW); 4599 Result r = t.get(g); 4600 assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0]))); 4601 4602 arm = new RowMutations(ROW); 4603 p = new Put(ROW); 4604 p.addColumn(FAMILY, QUALIFIERS[1], VALUE); 4605 arm.add(p); 4606 Delete d = new Delete(ROW); 4607 d.addColumns(FAMILY, QUALIFIERS[0]); 4608 arm.add(d); 4609 // TODO: Trying mutateRow again. The batch was failing with a one try only. 4610 t.mutateRow(arm); 4611 r = t.get(g); 4612 assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1]))); 4613 assertNull(r.getValue(FAMILY, QUALIFIERS[0])); 4614 4615 //Test that we get a region level exception 4616 try { 4617 arm = new RowMutations(ROW); 4618 p = new Put(ROW); 4619 p.addColumn(new byte[]{'b', 'o', 'g', 'u', 's'}, QUALIFIERS[0], VALUE); 4620 arm.add(p); 4621 t.mutateRow(arm); 4622 fail("Expected NoSuchColumnFamilyException"); 4623 } catch (RetriesExhaustedWithDetailsException e) { 4624 for (Throwable rootCause : e.getCauses()) { 4625 if (rootCause instanceof NoSuchColumnFamilyException) { 4626 return; 4627 } 4628 } 4629 throw e; 4630 } 4631 } 4632 } 4633 4634 @Test 4635 public void testBatchAppendWithReturnResultFalse() throws Exception { 4636 LOG.info("Starting testBatchAppendWithReturnResultFalse"); 4637 final TableName tableName = TableName.valueOf(name.getMethodName()); 4638 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) { 4639 Append append1 = new Append(Bytes.toBytes("row1")); 4640 append1.setReturnResults(false); 4641 append1.addColumn(FAMILY, Bytes.toBytes("f1"), Bytes.toBytes("value1")); 4642 Append append2 = new Append(Bytes.toBytes("row1")); 4643 append2.setReturnResults(false); 4644 append2.addColumn(FAMILY, Bytes.toBytes("f1"), Bytes.toBytes("value2")); 4645 List<Append> appends = new ArrayList<>(); 4646 appends.add(append1); 4647 appends.add(append2); 4648 Object[] results = new Object[2]; 4649 table.batch(appends, results); 4650 assertTrue(results.length == 2); 4651 for (Object r : results) { 4652 Result result = (Result) r; 4653 assertTrue(result.isEmpty()); 4654 } 4655 } 4656 } 4657 4658 @Test 4659 public void testAppend() throws Exception { 4660 LOG.info("Starting testAppend"); 4661 final TableName tableName = TableName.valueOf(name.getMethodName()); 4662 try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) { 4663 byte[] v1 = Bytes.toBytes("42"); 4664 byte[] v2 = Bytes.toBytes("23"); 4665 byte[][] QUALIFIERS = new byte[][]{ 4666 Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("c") 4667 }; 4668 Append a = new Append(ROW); 4669 a.addColumn(FAMILY, QUALIFIERS[0], v1); 4670 a.addColumn(FAMILY, QUALIFIERS[1], v2); 4671 a.setReturnResults(false); 4672 assertEmptyResult(t.append(a)); 4673 4674 a = new Append(ROW); 4675 a.addColumn(FAMILY, QUALIFIERS[0], v2); 4676 a.addColumn(FAMILY, QUALIFIERS[1], v1); 4677 a.addColumn(FAMILY, QUALIFIERS[2], v2); 4678 Result r = t.append(a); 4679 assertEquals(0, Bytes.compareTo(Bytes.add(v1, v2), r.getValue(FAMILY, QUALIFIERS[0]))); 4680 assertEquals(0, Bytes.compareTo(Bytes.add(v2, v1), r.getValue(FAMILY, QUALIFIERS[1]))); 4681 // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct 4682 assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2]))); 4683 assertEquals(r.getColumnLatestCell(FAMILY, QUALIFIERS[0]).getTimestamp(), 4684 r.getColumnLatestCell(FAMILY, QUALIFIERS[2]).getTimestamp()); 4685 } 4686 } 4687 private List<Result> doAppend(final boolean walUsed) throws IOException { 4688 LOG.info("Starting testAppend, walUsed is " + walUsed); 4689 final TableName TABLENAME = 4690 TableName.valueOf(walUsed ? "testAppendWithWAL" : "testAppendWithoutWAL"); 4691 try (Table t = TEST_UTIL.createTable(TABLENAME, FAMILY)) { 4692 final byte[] row1 = Bytes.toBytes("c"); 4693 final byte[] row2 = Bytes.toBytes("b"); 4694 final byte[] row3 = Bytes.toBytes("a"); 4695 final byte[] qual = Bytes.toBytes("qual"); 4696 Put put_0 = new Put(row2); 4697 put_0.addColumn(FAMILY, qual, Bytes.toBytes("put")); 4698 Put put_1 = new Put(row3); 4699 put_1.addColumn(FAMILY, qual, Bytes.toBytes("put")); 4700 Append append_0 = new Append(row1); 4701 append_0.addColumn(FAMILY, qual, Bytes.toBytes("i")); 4702 Append append_1 = new Append(row1); 4703 append_1.addColumn(FAMILY, qual, Bytes.toBytes("k")); 4704 Append append_2 = new Append(row1); 4705 append_2.addColumn(FAMILY, qual, Bytes.toBytes("e")); 4706 if (!walUsed) { 4707 append_2.setDurability(Durability.SKIP_WAL); 4708 } 4709 Append append_3 = new Append(row1); 4710 append_3.addColumn(FAMILY, qual, Bytes.toBytes("a")); 4711 Scan s = new Scan(); 4712 s.setCaching(1); 4713 t.append(append_0); 4714 t.put(put_0); 4715 t.put(put_1); 4716 List<Result> results = new LinkedList<>(); 4717 try (ResultScanner scanner = t.getScanner(s)) { 4718 t.append(append_1); 4719 t.append(append_2); 4720 t.append(append_3); 4721 for (Result r : scanner) { 4722 results.add(r); 4723 } 4724 } 4725 TEST_UTIL.deleteTable(TABLENAME); 4726 return results; 4727 } 4728 } 4729 4730 @Test 4731 public void testAppendWithoutWAL() throws Exception { 4732 List<Result> resultsWithWal = doAppend(true); 4733 List<Result> resultsWithoutWal = doAppend(false); 4734 assertEquals(resultsWithWal.size(), resultsWithoutWal.size()); 4735 for (int i = 0; i != resultsWithWal.size(); ++i) { 4736 Result resultWithWal = resultsWithWal.get(i); 4737 Result resultWithoutWal = resultsWithoutWal.get(i); 4738 assertEquals(resultWithWal.rawCells().length, resultWithoutWal.rawCells().length); 4739 for (int j = 0; j != resultWithWal.rawCells().length; ++j) { 4740 Cell cellWithWal = resultWithWal.rawCells()[j]; 4741 Cell cellWithoutWal = resultWithoutWal.rawCells()[j]; 4742 assertTrue(Bytes.equals(CellUtil.cloneRow(cellWithWal), 4743 CellUtil.cloneRow(cellWithoutWal))); 4744 assertTrue(Bytes.equals(CellUtil.cloneFamily(cellWithWal), 4745 CellUtil.cloneFamily(cellWithoutWal))); 4746 assertTrue(Bytes.equals(CellUtil.cloneQualifier(cellWithWal), 4747 CellUtil.cloneQualifier(cellWithoutWal))); 4748 assertTrue(Bytes.equals(CellUtil.cloneValue(cellWithWal), 4749 CellUtil.cloneValue(cellWithoutWal))); 4750 } 4751 } 4752 } 4753 4754 @Test 4755 public void testClientPoolRoundRobin() throws IOException { 4756 final TableName tableName = TableName.valueOf(name.getMethodName()); 4757 4758 int poolSize = 3; 4759 int numVersions = poolSize * 2; 4760 Configuration conf = TEST_UTIL.getConfiguration(); 4761 conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "round-robin"); 4762 conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize); 4763 4764 try (Table table = 4765 TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, Integer.MAX_VALUE)) { 4766 4767 final long ts = EnvironmentEdgeManager.currentTime(); 4768 Get get = new Get(ROW); 4769 get.addColumn(FAMILY, QUALIFIER); 4770 get.setMaxVersions(); 4771 4772 for (int versions = 1; versions <= numVersions; versions++) { 4773 Put put = new Put(ROW); 4774 put.addColumn(FAMILY, QUALIFIER, ts + versions, VALUE); 4775 table.put(put); 4776 4777 Result result = table.get(get); 4778 NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY) 4779 .get(QUALIFIER); 4780 4781 assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":" 4782 + Bytes.toString(QUALIFIER) + " did not match", versions, navigableMap.size()); 4783 for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) { 4784 assertTrue("The value at time " + entry.getKey() 4785 + " did not match what was put", 4786 Bytes.equals(VALUE, entry.getValue())); 4787 } 4788 } 4789 } 4790 } 4791 4792 @Ignore ("Flakey: HBASE-8989") @Test 4793 public void testClientPoolThreadLocal() throws IOException { 4794 final TableName tableName = TableName.valueOf(name.getMethodName()); 4795 4796 int poolSize = Integer.MAX_VALUE; 4797 int numVersions = 3; 4798 Configuration conf = TEST_UTIL.getConfiguration(); 4799 conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "thread-local"); 4800 conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize); 4801 4802 try (final Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 3)) { 4803 4804 final long ts = EnvironmentEdgeManager.currentTime(); 4805 final Get get = new Get(ROW); 4806 get.addColumn(FAMILY, QUALIFIER); 4807 get.setMaxVersions(); 4808 4809 for (int versions = 1; versions <= numVersions; versions++) { 4810 Put put = new Put(ROW); 4811 put.addColumn(FAMILY, QUALIFIER, ts + versions, VALUE); 4812 table.put(put); 4813 4814 Result result = table.get(get); 4815 NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY) 4816 .get(QUALIFIER); 4817 4818 assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":" 4819 + Bytes.toString(QUALIFIER) + " did not match", versions, navigableMap.size()); 4820 for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) { 4821 assertTrue("The value at time " + entry.getKey() 4822 + " did not match what was put", 4823 Bytes.equals(VALUE, entry.getValue())); 4824 } 4825 } 4826 4827 final Object waitLock = new Object(); 4828 ExecutorService executorService = Executors.newFixedThreadPool(numVersions); 4829 final AtomicReference<AssertionError> error = new AtomicReference<>(null); 4830 for (int versions = numVersions; versions < numVersions * 2; versions++) { 4831 final int versionsCopy = versions; 4832 executorService.submit(new Callable<Void>() { 4833 @Override 4834 public Void call() { 4835 try { 4836 Put put = new Put(ROW); 4837 put.addColumn(FAMILY, QUALIFIER, ts + versionsCopy, VALUE); 4838 table.put(put); 4839 4840 Result result = table.get(get); 4841 NavigableMap<Long, byte[]> navigableMap = result.getMap() 4842 .get(FAMILY).get(QUALIFIER); 4843 4844 assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":" 4845 + Bytes.toString(QUALIFIER) + " did not match " + versionsCopy, versionsCopy, 4846 navigableMap.size()); 4847 for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) { 4848 assertTrue("The value at time " + entry.getKey() 4849 + " did not match what was put", 4850 Bytes.equals(VALUE, entry.getValue())); 4851 } 4852 synchronized (waitLock) { 4853 waitLock.wait(); 4854 } 4855 } catch (Exception e) { 4856 } catch (AssertionError e) { 4857 // the error happens in a thread, it won't fail the test, 4858 // need to pass it to the caller for proper handling. 4859 error.set(e); 4860 LOG.error(e.toString(), e); 4861 } 4862 4863 return null; 4864 } 4865 }); 4866 } 4867 synchronized (waitLock) { 4868 waitLock.notifyAll(); 4869 } 4870 executorService.shutdownNow(); 4871 assertNull(error.get()); 4872 } 4873 } 4874 4875 @Test 4876 public void testCheckAndPut() throws IOException { 4877 final byte [] anotherrow = Bytes.toBytes("anotherrow"); 4878 final byte [] value2 = Bytes.toBytes("abcd"); 4879 4880 try (Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY)) { 4881 Put put1 = new Put(ROW); 4882 put1.addColumn(FAMILY, QUALIFIER, VALUE); 4883 4884 // row doesn't exist, so using non-null value should be considered "not match". 4885 boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 4886 .ifEquals(VALUE).thenPut(put1); 4887 assertFalse(ok); 4888 4889 // row doesn't exist, so using "ifNotExists" should be considered "match". 4890 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put1); 4891 assertTrue(ok); 4892 4893 // row now exists, so using "ifNotExists" should be considered "not match". 4894 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put1); 4895 assertFalse(ok); 4896 4897 Put put2 = new Put(ROW); 4898 put2.addColumn(FAMILY, QUALIFIER, value2); 4899 4900 // row now exists, use the matching value to check 4901 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifEquals(VALUE).thenPut(put2); 4902 assertTrue(ok); 4903 4904 Put put3 = new Put(anotherrow); 4905 put3.addColumn(FAMILY, QUALIFIER, VALUE); 4906 4907 // try to do CheckAndPut on different rows 4908 try { 4909 table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifEquals(value2).thenPut(put3); 4910 fail("trying to check and modify different rows should have failed."); 4911 } catch (Exception e) { 4912 } 4913 } 4914 } 4915 4916 @Test 4917 public void testCheckAndMutateWithTimeRange() throws IOException { 4918 try (Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY)) { 4919 final long ts = System.currentTimeMillis() / 2; 4920 Put put = new Put(ROW); 4921 put.addColumn(FAMILY, QUALIFIER, ts, VALUE); 4922 4923 boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 4924 .ifNotExists() 4925 .thenPut(put); 4926 assertTrue(ok); 4927 4928 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 4929 .timeRange(TimeRange.at(ts + 10000)) 4930 .ifEquals(VALUE) 4931 .thenPut(put); 4932 assertFalse(ok); 4933 4934 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 4935 .timeRange(TimeRange.from(ts + 10000)) 4936 .ifEquals(VALUE) 4937 .thenPut(put); 4938 assertFalse(ok); 4939 4940 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 4941 .timeRange(TimeRange.between(ts + 10000, ts + 20000)) 4942 .ifEquals(VALUE) 4943 .thenPut(put); 4944 assertFalse(ok); 4945 4946 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 4947 .timeRange(TimeRange.until(ts)) 4948 .ifEquals(VALUE) 4949 .thenPut(put); 4950 assertFalse(ok); 4951 4952 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 4953 .timeRange(TimeRange.at(ts)) 4954 .ifEquals(VALUE) 4955 .thenPut(put); 4956 assertTrue(ok); 4957 4958 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 4959 .timeRange(TimeRange.from(ts)) 4960 .ifEquals(VALUE) 4961 .thenPut(put); 4962 assertTrue(ok); 4963 4964 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 4965 .timeRange(TimeRange.between(ts, ts + 20000)) 4966 .ifEquals(VALUE) 4967 .thenPut(put); 4968 assertTrue(ok); 4969 4970 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 4971 .timeRange(TimeRange.until(ts + 10000)) 4972 .ifEquals(VALUE) 4973 .thenPut(put); 4974 assertTrue(ok); 4975 4976 RowMutations rm = new RowMutations(ROW) 4977 .add((Mutation) put); 4978 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 4979 .timeRange(TimeRange.at(ts + 10000)) 4980 .ifEquals(VALUE) 4981 .thenMutate(rm); 4982 assertFalse(ok); 4983 4984 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 4985 .timeRange(TimeRange.at(ts)) 4986 .ifEquals(VALUE) 4987 .thenMutate(rm); 4988 assertTrue(ok); 4989 4990 Delete delete = new Delete(ROW) 4991 .addColumn(FAMILY, QUALIFIER); 4992 4993 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 4994 .timeRange(TimeRange.at(ts + 10000)) 4995 .ifEquals(VALUE) 4996 .thenDelete(delete); 4997 assertFalse(ok); 4998 4999 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5000 .timeRange(TimeRange.at(ts)) 5001 .ifEquals(VALUE) 5002 .thenDelete(delete); 5003 assertTrue(ok); 5004 } 5005 } 5006 5007 @Test 5008 public void testCheckAndPutWithCompareOp() throws IOException { 5009 final byte [] value1 = Bytes.toBytes("aaaa"); 5010 final byte [] value2 = Bytes.toBytes("bbbb"); 5011 final byte [] value3 = Bytes.toBytes("cccc"); 5012 final byte [] value4 = Bytes.toBytes("dddd"); 5013 5014 try (Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY)) { 5015 5016 Put put2 = new Put(ROW); 5017 put2.addColumn(FAMILY, QUALIFIER, value2); 5018 5019 Put put3 = new Put(ROW); 5020 put3.addColumn(FAMILY, QUALIFIER, value3); 5021 5022 // row doesn't exist, so using "ifNotExists" should be considered "match". 5023 boolean ok = 5024 table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put2); 5025 assertTrue(ok); 5026 5027 // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL 5028 // turns out "match" 5029 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5030 .ifMatches(CompareOperator.GREATER, value1).thenPut(put2); 5031 assertFalse(ok); 5032 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5033 .ifMatches(CompareOperator.EQUAL, value1).thenPut(put2); 5034 assertFalse(ok); 5035 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5036 .ifMatches(CompareOperator.GREATER_OR_EQUAL, value1).thenPut(put2); 5037 assertFalse(ok); 5038 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5039 .ifMatches(CompareOperator.LESS, value1).thenPut(put2); 5040 assertTrue(ok); 5041 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5042 .ifMatches(CompareOperator.LESS_OR_EQUAL, value1).thenPut(put2); 5043 assertTrue(ok); 5044 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5045 .ifMatches(CompareOperator.NOT_EQUAL, value1).thenPut(put3); 5046 assertTrue(ok); 5047 5048 // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL 5049 // turns out "match" 5050 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5051 .ifMatches(CompareOperator.LESS, value4).thenPut(put3); 5052 assertFalse(ok); 5053 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5054 .ifMatches(CompareOperator.LESS_OR_EQUAL, value4).thenPut(put3); 5055 assertFalse(ok); 5056 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5057 .ifMatches(CompareOperator.EQUAL, value4).thenPut(put3); 5058 assertFalse(ok); 5059 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5060 .ifMatches(CompareOperator.GREATER, value4).thenPut(put3); 5061 assertTrue(ok); 5062 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5063 .ifMatches(CompareOperator.GREATER_OR_EQUAL, value4).thenPut(put3); 5064 assertTrue(ok); 5065 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5066 .ifMatches(CompareOperator.NOT_EQUAL, value4).thenPut(put2); 5067 assertTrue(ok); 5068 5069 // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL 5070 // turns out "match" 5071 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5072 .ifMatches(CompareOperator.GREATER, value2).thenPut(put2); 5073 assertFalse(ok); 5074 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5075 .ifMatches(CompareOperator.NOT_EQUAL, value2).thenPut(put2); 5076 assertFalse(ok); 5077 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5078 .ifMatches(CompareOperator.LESS, value2).thenPut(put2); 5079 assertFalse(ok); 5080 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5081 .ifMatches(CompareOperator.GREATER_OR_EQUAL, value2).thenPut(put2); 5082 assertTrue(ok); 5083 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5084 .ifMatches(CompareOperator.LESS_OR_EQUAL, value2).thenPut(put2); 5085 assertTrue(ok); 5086 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5087 .ifMatches(CompareOperator.EQUAL, value2).thenPut(put3); 5088 assertTrue(ok); 5089 } 5090 } 5091 5092 @Test 5093 public void testCheckAndDelete() throws IOException { 5094 final byte [] value1 = Bytes.toBytes("aaaa"); 5095 5096 try (Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), 5097 FAMILY)) { 5098 5099 Put put = new Put(ROW); 5100 put.addColumn(FAMILY, QUALIFIER, value1); 5101 table.put(put); 5102 5103 Delete delete = new Delete(ROW); 5104 delete.addColumns(FAMILY, QUALIFIER); 5105 5106 boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5107 .ifEquals(value1).thenDelete(delete); 5108 assertTrue(ok); 5109 } 5110 } 5111 5112 @Test 5113 public void testCheckAndDeleteWithCompareOp() throws IOException { 5114 final byte [] value1 = Bytes.toBytes("aaaa"); 5115 final byte [] value2 = Bytes.toBytes("bbbb"); 5116 final byte [] value3 = Bytes.toBytes("cccc"); 5117 final byte [] value4 = Bytes.toBytes("dddd"); 5118 5119 try (Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), 5120 FAMILY)) { 5121 5122 Put put2 = new Put(ROW); 5123 put2.addColumn(FAMILY, QUALIFIER, value2); 5124 table.put(put2); 5125 5126 Put put3 = new Put(ROW); 5127 put3.addColumn(FAMILY, QUALIFIER, value3); 5128 5129 Delete delete = new Delete(ROW); 5130 delete.addColumns(FAMILY, QUALIFIER); 5131 5132 // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL 5133 // turns out "match" 5134 boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5135 .ifMatches(CompareOperator.GREATER, value1).thenDelete(delete); 5136 assertFalse(ok); 5137 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5138 .ifMatches(CompareOperator.EQUAL, value1).thenDelete(delete); 5139 assertFalse(ok); 5140 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5141 .ifMatches(CompareOperator.GREATER_OR_EQUAL, value1).thenDelete(delete); 5142 assertFalse(ok); 5143 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5144 .ifMatches(CompareOperator.LESS, value1).thenDelete(delete); 5145 assertTrue(ok); 5146 table.put(put2); 5147 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5148 .ifMatches(CompareOperator.LESS_OR_EQUAL, value1).thenDelete(delete); 5149 assertTrue(ok); 5150 table.put(put2); 5151 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5152 .ifMatches(CompareOperator.NOT_EQUAL, value1).thenDelete(delete); 5153 assertTrue(ok); 5154 5155 // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL 5156 // turns out "match" 5157 table.put(put3); 5158 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5159 .ifMatches(CompareOperator.LESS, value4).thenDelete(delete); 5160 assertFalse(ok); 5161 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5162 .ifMatches(CompareOperator.LESS_OR_EQUAL, value4).thenDelete(delete); 5163 assertFalse(ok); 5164 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5165 .ifMatches(CompareOperator.EQUAL, value4).thenDelete(delete); 5166 assertFalse(ok); 5167 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5168 .ifMatches(CompareOperator.GREATER, value4).thenDelete(delete); 5169 assertTrue(ok); 5170 table.put(put3); 5171 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5172 .ifMatches(CompareOperator.GREATER_OR_EQUAL, value4).thenDelete(delete); 5173 assertTrue(ok); 5174 table.put(put3); 5175 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5176 .ifMatches(CompareOperator.NOT_EQUAL, value4).thenDelete(delete); 5177 assertTrue(ok); 5178 5179 // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL 5180 // turns out "match" 5181 table.put(put2); 5182 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5183 .ifMatches(CompareOperator.GREATER, value2).thenDelete(delete); 5184 assertFalse(ok); 5185 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5186 .ifMatches(CompareOperator.NOT_EQUAL, value2).thenDelete(delete); 5187 assertFalse(ok); 5188 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5189 .ifMatches(CompareOperator.LESS, value2).thenDelete(delete); 5190 assertFalse(ok); 5191 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5192 .ifMatches(CompareOperator.GREATER_OR_EQUAL, value2).thenDelete(delete); 5193 assertTrue(ok); 5194 table.put(put2); 5195 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5196 .ifMatches(CompareOperator.LESS_OR_EQUAL, value2).thenDelete(delete); 5197 assertTrue(ok); 5198 table.put(put2); 5199 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER) 5200 .ifMatches(CompareOperator.EQUAL, value2).thenDelete(delete); 5201 assertTrue(ok); 5202 } 5203 } 5204 5205 /** 5206 * Test ScanMetrics 5207 */ 5208 @Test 5209 @SuppressWarnings ("unused") 5210 public void testScanMetrics() throws Exception { 5211 final TableName tableName = TableName.valueOf(name.getMethodName()); 5212 5213 // Set up test table: 5214 // Create table: 5215 try (Table ht = TEST_UTIL.createMultiRegionTable(tableName, FAMILY)) { 5216 int numOfRegions = -1; 5217 try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) { 5218 numOfRegions = r.getStartKeys().length; 5219 } 5220 // Create 3 rows in the table, with rowkeys starting with "zzz*" so that 5221 // scan are forced to hit all the regions. 5222 Put put1 = new Put(Bytes.toBytes("zzz1")); 5223 put1.addColumn(FAMILY, QUALIFIER, VALUE); 5224 Put put2 = new Put(Bytes.toBytes("zzz2")); 5225 put2.addColumn(FAMILY, QUALIFIER, VALUE); 5226 Put put3 = new Put(Bytes.toBytes("zzz3")); 5227 put3.addColumn(FAMILY, QUALIFIER, VALUE); 5228 ht.put(Arrays.asList(put1, put2, put3)); 5229 5230 Scan scan1 = new Scan(); 5231 int numRecords = 0; 5232 try (ResultScanner scanner = ht.getScanner(scan1)) { 5233 for (Result result : scanner) { 5234 numRecords++; 5235 } 5236 5237 LOG.info("test data has " + numRecords + " records."); 5238 5239 // by default, scan metrics collection is turned off 5240 assertEquals(null, scanner.getScanMetrics()); 5241 } 5242 5243 // turn on scan metrics 5244 Scan scan2 = new Scan(); 5245 scan2.setScanMetricsEnabled(true); 5246 scan2.setCaching(numRecords + 1); 5247 try (ResultScanner scanner = ht.getScanner(scan2)) { 5248 for (Result result : scanner.next(numRecords - 1)) { 5249 } 5250 } 5251 // closing the scanner will set the metrics. 5252 assertNotNull(scan2.getScanMetrics()); 5253 5254 // set caching to 1, because metrics are collected in each roundtrip only 5255 scan2 = new Scan(); 5256 scan2.setScanMetricsEnabled(true); 5257 scan2.setCaching(1); 5258 try (ResultScanner scanner = ht.getScanner(scan2)) { 5259 // per HBASE-5717, this should still collect even if you don't run all the way to 5260 // the end of the scanner. So this is asking for 2 of the 3 rows we inserted. 5261 for (Result result : scanner.next(numRecords - 1)) { 5262 } 5263 } 5264 ScanMetrics scanMetrics = scan2.getScanMetrics(); 5265 assertEquals("Did not access all the regions in the table", numOfRegions, 5266 scanMetrics.countOfRegions.get()); 5267 5268 // check byte counters 5269 scan2 = new Scan(); 5270 scan2.setScanMetricsEnabled(true); 5271 scan2.setCaching(1); 5272 try (ResultScanner scanner = ht.getScanner(scan2)) { 5273 int numBytes = 0; 5274 for (Result result : scanner.next(1)) { 5275 for (Cell cell : result.listCells()) { 5276 numBytes += PrivateCellUtil.estimatedSerializedSizeOf(cell); 5277 } 5278 } 5279 scanMetrics = scanner.getScanMetrics(); 5280 assertEquals("Did not count the result bytes", numBytes, 5281 scanMetrics.countOfBytesInResults.get()); 5282 } 5283 5284 // check byte counters on a small scan 5285 scan2 = new Scan(); 5286 scan2.setScanMetricsEnabled(true); 5287 scan2.setCaching(1); 5288 scan2.setSmall(true); 5289 try (ResultScanner scanner = ht.getScanner(scan2)) { 5290 int numBytes = 0; 5291 for (Result result : scanner.next(1)) { 5292 for (Cell cell : result.listCells()) { 5293 numBytes += PrivateCellUtil.estimatedSerializedSizeOf(cell); 5294 } 5295 } 5296 scanMetrics = scanner.getScanMetrics(); 5297 assertEquals("Did not count the result bytes", numBytes, 5298 scanMetrics.countOfBytesInResults.get()); 5299 } 5300 5301 // now, test that the metrics are still collected even if you don't call close, but do 5302 // run past the end of all the records 5303 /** There seems to be a timing issue here. Comment out for now. Fix when time. 5304 Scan scanWithoutClose = new Scan(); 5305 scanWithoutClose.setCaching(1); 5306 scanWithoutClose.setScanMetricsEnabled(true); 5307 ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose); 5308 for (Result result : scannerWithoutClose.next(numRecords + 1)) { 5309 } 5310 ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose); 5311 assertEquals("Did not access all the regions in the table", numOfRegions, 5312 scanMetricsWithoutClose.countOfRegions.get()); 5313 */ 5314 5315 // finally, 5316 // test that the metrics are collected correctly if you both run past all the records, 5317 // AND close the scanner 5318 Scan scanWithClose = new Scan(); 5319 // make sure we can set caching up to the number of a scanned values 5320 scanWithClose.setCaching(numRecords); 5321 scanWithClose.setScanMetricsEnabled(true); 5322 try (ResultScanner scannerWithClose = ht.getScanner(scanWithClose)) { 5323 for (Result result : scannerWithClose.next(numRecords + 1)) { 5324 } 5325 } 5326 ScanMetrics scanMetricsWithClose = scanWithClose.getScanMetrics(); 5327 assertEquals("Did not access all the regions in the table", numOfRegions, 5328 scanMetricsWithClose.countOfRegions.get()); 5329 } 5330 } 5331 5332 private ScanMetrics getScanMetrics(Scan scan) throws Exception { 5333 byte[] serializedMetrics = scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA); 5334 assertTrue("Serialized metrics were not found.", serializedMetrics != null); 5335 5336 ScanMetrics scanMetrics = ProtobufUtil.toScanMetrics(serializedMetrics); 5337 5338 return scanMetrics; 5339 } 5340 5341 /** 5342 * Tests that cache on write works all the way up from the client-side. 5343 * 5344 * Performs inserts, flushes, and compactions, verifying changes in the block 5345 * cache along the way. 5346 */ 5347 @Test 5348 public void testCacheOnWriteEvictOnClose() throws Exception { 5349 final TableName tableName = TableName.valueOf(name.getMethodName()); 5350 byte [] data = Bytes.toBytes("data"); 5351 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) { 5352 try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { 5353 // get the block cache and region 5354 String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName(); 5355 5356 HRegion region = TEST_UTIL.getRSForFirstRegionInTable(tableName) 5357 .getRegion(regionName); 5358 HStore store = region.getStores().iterator().next(); 5359 CacheConfig cacheConf = store.getCacheConfig(); 5360 cacheConf.setCacheDataOnWrite(true); 5361 cacheConf.setEvictOnClose(true); 5362 BlockCache cache = cacheConf.getBlockCache(); 5363 5364 // establish baseline stats 5365 long startBlockCount = cache.getBlockCount(); 5366 long startBlockHits = cache.getStats().getHitCount(); 5367 long startBlockMiss = cache.getStats().getMissCount(); 5368 5369 // wait till baseline is stable, (minimal 500 ms) 5370 for (int i = 0; i < 5; i++) { 5371 Thread.sleep(100); 5372 if (startBlockCount != cache.getBlockCount() 5373 || startBlockHits != cache.getStats().getHitCount() 5374 || startBlockMiss != cache.getStats().getMissCount()) { 5375 startBlockCount = cache.getBlockCount(); 5376 startBlockHits = cache.getStats().getHitCount(); 5377 startBlockMiss = cache.getStats().getMissCount(); 5378 i = -1; 5379 } 5380 } 5381 5382 // insert data 5383 Put put = new Put(ROW); 5384 put.addColumn(FAMILY, QUALIFIER, data); 5385 table.put(put); 5386 assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data)); 5387 // data was in memstore so don't expect any changes 5388 assertEquals(startBlockCount, cache.getBlockCount()); 5389 assertEquals(startBlockHits, cache.getStats().getHitCount()); 5390 assertEquals(startBlockMiss, cache.getStats().getMissCount()); 5391 // flush the data 5392 System.out.println("Flushing cache"); 5393 region.flush(true); 5394 // expect one more block in cache, no change in hits/misses 5395 long expectedBlockCount = startBlockCount + 1; 5396 long expectedBlockHits = startBlockHits; 5397 long expectedBlockMiss = startBlockMiss; 5398 assertEquals(expectedBlockCount, cache.getBlockCount()); 5399 assertEquals(expectedBlockHits, cache.getStats().getHitCount()); 5400 assertEquals(expectedBlockMiss, cache.getStats().getMissCount()); 5401 // read the data and expect same blocks, one new hit, no misses 5402 assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data)); 5403 assertEquals(expectedBlockCount, cache.getBlockCount()); 5404 assertEquals(++expectedBlockHits, cache.getStats().getHitCount()); 5405 assertEquals(expectedBlockMiss, cache.getStats().getMissCount()); 5406 // insert a second column, read the row, no new blocks, one new hit 5407 byte[] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER); 5408 byte[] data2 = Bytes.add(data, data); 5409 put = new Put(ROW); 5410 put.addColumn(FAMILY, QUALIFIER2, data2); 5411 table.put(put); 5412 Result r = table.get(new Get(ROW)); 5413 assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data)); 5414 assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2)); 5415 assertEquals(expectedBlockCount, cache.getBlockCount()); 5416 assertEquals(++expectedBlockHits, cache.getStats().getHitCount()); 5417 assertEquals(expectedBlockMiss, cache.getStats().getMissCount()); 5418 // flush, one new block 5419 System.out.println("Flushing cache"); 5420 region.flush(true); 5421 assertEquals(++expectedBlockCount, cache.getBlockCount()); 5422 assertEquals(expectedBlockHits, cache.getStats().getHitCount()); 5423 assertEquals(expectedBlockMiss, cache.getStats().getMissCount()); 5424 // compact, net minus two blocks, two hits, no misses 5425 System.out.println("Compacting"); 5426 assertEquals(2, store.getStorefilesCount()); 5427 store.triggerMajorCompaction(); 5428 region.compact(true); 5429 store.closeAndArchiveCompactedFiles(); 5430 waitForStoreFileCount(store, 1, 10000); // wait 10 seconds max 5431 assertEquals(1, store.getStorefilesCount()); 5432 expectedBlockCount -= 2; // evicted two blocks, cached none 5433 assertEquals(expectedBlockCount, cache.getBlockCount()); 5434 expectedBlockHits += 2; 5435 assertEquals(expectedBlockMiss, cache.getStats().getMissCount()); 5436 assertEquals(expectedBlockHits, cache.getStats().getHitCount()); 5437 // read the row, this should be a cache miss because we don't cache data 5438 // blocks on compaction 5439 r = table.get(new Get(ROW)); 5440 assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data)); 5441 assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2)); 5442 expectedBlockCount += 1; // cached one data block 5443 assertEquals(expectedBlockCount, cache.getBlockCount()); 5444 assertEquals(expectedBlockHits, cache.getStats().getHitCount()); 5445 assertEquals(++expectedBlockMiss, cache.getStats().getMissCount()); 5446 } 5447 } 5448 } 5449 5450 private void waitForStoreFileCount(HStore store, int count, int timeout) 5451 throws InterruptedException { 5452 long start = System.currentTimeMillis(); 5453 while (start + timeout > System.currentTimeMillis() && store.getStorefilesCount() != count) { 5454 Thread.sleep(100); 5455 } 5456 System.out.println("start=" + start + ", now=" + System.currentTimeMillis() + ", cur=" + 5457 store.getStorefilesCount()); 5458 assertEquals(count, store.getStorefilesCount()); 5459 } 5460 5461 @Test 5462 /** 5463 * Tests the non cached version of getRegionLocator by moving a region. 5464 */ 5465 public void testNonCachedGetRegionLocation() throws Exception { 5466 // Test Initialization. 5467 final TableName tableName = TableName.valueOf(name.getMethodName()); 5468 byte [] family1 = Bytes.toBytes("f1"); 5469 byte [] family2 = Bytes.toBytes("f2"); 5470 try (Table table = TEST_UTIL.createTable(tableName, new byte[][] {family1, family2}, 10); 5471 Admin admin = TEST_UTIL.getAdmin(); 5472 RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { 5473 List<HRegionLocation> allRegionLocations = locator.getAllRegionLocations(); 5474 assertEquals(1, allRegionLocations.size()); 5475 HRegionInfo regionInfo = allRegionLocations.get(0).getRegionInfo(); 5476 ServerName addrBefore = allRegionLocations.get(0).getServerName(); 5477 // Verify region location before move. 5478 HRegionLocation addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false); 5479 HRegionLocation addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(), true); 5480 5481 assertEquals(addrBefore.getPort(), addrCache.getPort()); 5482 assertEquals(addrBefore.getPort(), addrNoCache.getPort()); 5483 5484 ServerName addrAfter = null; 5485 // Now move the region to a different server. 5486 for (int i = 0; i < SLAVES; i++) { 5487 HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i); 5488 ServerName addr = regionServer.getServerName(); 5489 if (addr.getPort() != addrBefore.getPort()) { 5490 admin.move(regionInfo.getEncodedNameAsBytes(), 5491 Bytes.toBytes(addr.toString())); 5492 // Wait for the region to move. 5493 Thread.sleep(5000); 5494 addrAfter = addr; 5495 break; 5496 } 5497 } 5498 5499 // Verify the region was moved. 5500 addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false); 5501 addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(), true); 5502 assertNotNull(addrAfter); 5503 assertTrue(addrAfter.getPort() != addrCache.getPort()); 5504 assertEquals(addrAfter.getPort(), addrNoCache.getPort()); 5505 } 5506 } 5507 5508 @Test 5509 /** 5510 * Tests getRegionsInRange by creating some regions over which a range of 5511 * keys spans; then changing the key range. 5512 */ 5513 public void testGetRegionsInRange() throws Exception { 5514 // Test Initialization. 5515 byte [] startKey = Bytes.toBytes("ddc"); 5516 byte [] endKey = Bytes.toBytes("mmm"); 5517 TableName tableName = TableName.valueOf(name.getMethodName()); 5518 TEST_UTIL.createMultiRegionTable(tableName, new byte[][] { FAMILY }, 10); 5519 5520 int numOfRegions = -1; 5521 try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) { 5522 numOfRegions = r.getStartKeys().length; 5523 } 5524 assertEquals(26, numOfRegions); 5525 5526 // Get the regions in this range 5527 List<HRegionLocation> regionsList = getRegionsInRange(tableName, startKey, endKey); 5528 assertEquals(10, regionsList.size()); 5529 5530 // Change the start key 5531 startKey = Bytes.toBytes("fff"); 5532 regionsList = getRegionsInRange(tableName, startKey, endKey); 5533 assertEquals(7, regionsList.size()); 5534 5535 // Change the end key 5536 endKey = Bytes.toBytes("nnn"); 5537 regionsList = getRegionsInRange(tableName, startKey, endKey); 5538 assertEquals(8, regionsList.size()); 5539 5540 // Empty start key 5541 regionsList = getRegionsInRange(tableName, HConstants.EMPTY_START_ROW, endKey); 5542 assertEquals(13, regionsList.size()); 5543 5544 // Empty end key 5545 regionsList = getRegionsInRange(tableName, startKey, HConstants.EMPTY_END_ROW); 5546 assertEquals(21, regionsList.size()); 5547 5548 // Both start and end keys empty 5549 regionsList = getRegionsInRange(tableName, HConstants.EMPTY_START_ROW, 5550 HConstants.EMPTY_END_ROW); 5551 assertEquals(26, regionsList.size()); 5552 5553 // Change the end key to somewhere in the last block 5554 endKey = Bytes.toBytes("zzz1"); 5555 regionsList = getRegionsInRange(tableName, startKey, endKey); 5556 assertEquals(21, regionsList.size()); 5557 5558 // Change the start key to somewhere in the first block 5559 startKey = Bytes.toBytes("aac"); 5560 regionsList = getRegionsInRange(tableName, startKey, endKey); 5561 assertEquals(26, regionsList.size()); 5562 5563 // Make start and end key the same 5564 startKey = endKey = Bytes.toBytes("ccc"); 5565 regionsList = getRegionsInRange(tableName, startKey, endKey); 5566 assertEquals(1, regionsList.size()); 5567 } 5568 5569 private List<HRegionLocation> getRegionsInRange(TableName tableName, byte[] startKey, 5570 byte[] endKey) throws IOException { 5571 List<HRegionLocation> regionsInRange = new ArrayList<>(); 5572 byte[] currentKey = startKey; 5573 final boolean endKeyIsEndOfTable = Bytes.equals(endKey, HConstants.EMPTY_END_ROW); 5574 try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) { 5575 do { 5576 HRegionLocation regionLocation = r.getRegionLocation(currentKey); 5577 regionsInRange.add(regionLocation); 5578 currentKey = regionLocation.getRegionInfo().getEndKey(); 5579 } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW) 5580 && (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0)); 5581 return regionsInRange; 5582 } 5583 } 5584 5585 @Test 5586 public void testJira6912() throws Exception { 5587 final TableName tableName = TableName.valueOf(name.getMethodName()); 5588 try (Table foo = TEST_UTIL.createTable(tableName, new byte[][] {FAMILY}, 10)) { 5589 5590 List<Put> puts = new ArrayList<Put>(); 5591 for (int i = 0; i != 100; i++) { 5592 Put put = new Put(Bytes.toBytes(i)); 5593 put.addColumn(FAMILY, FAMILY, Bytes.toBytes(i)); 5594 puts.add(put); 5595 } 5596 foo.put(puts); 5597 // If i comment this out it works 5598 TEST_UTIL.flush(); 5599 5600 Scan scan = new Scan(); 5601 scan.setStartRow(Bytes.toBytes(1)); 5602 scan.setStopRow(Bytes.toBytes(3)); 5603 scan.addColumn(FAMILY, FAMILY); 5604 scan.setFilter(new RowFilter(CompareOperator.NOT_EQUAL, 5605 new BinaryComparator(Bytes.toBytes(1)))); 5606 5607 try (ResultScanner scanner = foo.getScanner(scan)) { 5608 Result[] bar = scanner.next(100); 5609 assertEquals(1, bar.length); 5610 } 5611 } 5612 } 5613 5614 @Test 5615 public void testScan_NullQualifier() throws IOException { 5616 try (Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY)) { 5617 Put put = new Put(ROW); 5618 put.addColumn(FAMILY, QUALIFIER, VALUE); 5619 table.put(put); 5620 5621 put = new Put(ROW); 5622 put.addColumn(FAMILY, null, VALUE); 5623 table.put(put); 5624 LOG.info("Row put"); 5625 5626 Scan scan = new Scan(); 5627 scan.addColumn(FAMILY, null); 5628 5629 ResultScanner scanner = table.getScanner(scan); 5630 Result[] bar = scanner.next(100); 5631 assertEquals(1, bar.length); 5632 assertEquals(1, bar[0].size()); 5633 5634 scan = new Scan(); 5635 scan.addFamily(FAMILY); 5636 5637 scanner = table.getScanner(scan); 5638 bar = scanner.next(100); 5639 assertEquals(1, bar.length); 5640 assertEquals(2, bar[0].size()); 5641 } 5642 } 5643 5644 @Test 5645 public void testNegativeTimestamp() throws IOException { 5646 try (Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY)) { 5647 5648 try { 5649 Put put = new Put(ROW, -1); 5650 put.addColumn(FAMILY, QUALIFIER, VALUE); 5651 table.put(put); 5652 fail("Negative timestamps should not have been allowed"); 5653 } catch (IllegalArgumentException ex) { 5654 assertTrue(ex.getMessage().contains("negative")); 5655 } 5656 5657 try { 5658 Put put = new Put(ROW); 5659 long ts = -1; 5660 put.addColumn(FAMILY, QUALIFIER, ts, VALUE); 5661 table.put(put); 5662 fail("Negative timestamps should not have been allowed"); 5663 } catch (IllegalArgumentException ex) { 5664 assertTrue(ex.getMessage().contains("negative")); 5665 } 5666 5667 try { 5668 Delete delete = new Delete(ROW, -1); 5669 table.delete(delete); 5670 fail("Negative timestamps should not have been allowed"); 5671 } catch (IllegalArgumentException ex) { 5672 assertTrue(ex.getMessage().contains("negative")); 5673 } 5674 5675 try { 5676 Delete delete = new Delete(ROW); 5677 delete.addFamily(FAMILY, -1); 5678 table.delete(delete); 5679 fail("Negative timestamps should not have been allowed"); 5680 } catch (IllegalArgumentException ex) { 5681 assertTrue(ex.getMessage().contains("negative")); 5682 } 5683 5684 try { 5685 Scan scan = new Scan(); 5686 scan.setTimeRange(-1, 1); 5687 table.getScanner(scan); 5688 fail("Negative timestamps should not have been allowed"); 5689 } catch (IllegalArgumentException ex) { 5690 assertTrue(ex.getMessage().contains("negative")); 5691 } 5692 5693 // KeyValue should allow negative timestamps for backwards compat. Otherwise, if the user 5694 // already has negative timestamps in cluster data, HBase won't be able to handle that 5695 try { 5696 new KeyValue(Bytes.toBytes(42), Bytes.toBytes(42), Bytes.toBytes(42), -1, 5697 Bytes.toBytes(42)); 5698 } catch (IllegalArgumentException ex) { 5699 fail("KeyValue SHOULD allow negative timestamps"); 5700 } 5701 5702 } 5703 } 5704 5705 @Test 5706 public void testRawScanRespectsVersions() throws Exception { 5707 final TableName tableName = TableName.valueOf(name.getMethodName()); 5708 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) { 5709 byte[] row = Bytes.toBytes("row"); 5710 5711 // put the same row 4 times, with different values 5712 Put p = new Put(row); 5713 p.addColumn(FAMILY, QUALIFIER, 10, VALUE); 5714 table.put(p); 5715 p = new Put(row); 5716 p.addColumn(FAMILY, QUALIFIER, 11, ArrayUtils.add(VALUE, (byte) 2)); 5717 table.put(p); 5718 5719 p = new Put(row); 5720 p.addColumn(FAMILY, QUALIFIER, 12, ArrayUtils.add(VALUE, (byte) 3)); 5721 table.put(p); 5722 5723 p = new Put(row); 5724 p.addColumn(FAMILY, QUALIFIER, 13, ArrayUtils.add(VALUE, (byte) 4)); 5725 table.put(p); 5726 5727 int versions = 4; 5728 Scan s = new Scan(row); 5729 // get all the possible versions 5730 s.setMaxVersions(); 5731 s.setRaw(true); 5732 5733 try (ResultScanner scanner = table.getScanner(s)) { 5734 int count = 0; 5735 for (Result r : scanner) { 5736 assertEquals("Found an unexpected number of results for the row!", versions, 5737 r.listCells().size()); 5738 count++; 5739 } 5740 assertEquals("Found more than a single row when raw scanning the table with a single row!", 5741 1, count); 5742 } 5743 5744 // then if we decrease the number of versions, but keep the scan raw, we should see exactly 5745 // that number of versions 5746 versions = 2; 5747 s.setMaxVersions(versions); 5748 try (ResultScanner scanner = table.getScanner(s)) { 5749 int count = 0; 5750 for (Result r : scanner) { 5751 assertEquals("Found an unexpected number of results for the row!", versions, 5752 r.listCells().size()); 5753 count++; 5754 } 5755 assertEquals("Found more than a single row when raw scanning the table with a single row!", 5756 1, count); 5757 } 5758 5759 // finally, if we turn off raw scanning, but max out the number of versions, we should go back 5760 // to seeing just three 5761 versions = 3; 5762 s.setMaxVersions(versions); 5763 try (ResultScanner scanner = table.getScanner(s)) { 5764 int count = 0; 5765 for (Result r : scanner) { 5766 assertEquals("Found an unexpected number of results for the row!", versions, 5767 r.listCells().size()); 5768 count++; 5769 } 5770 assertEquals("Found more than a single row when raw scanning the table with a single row!", 5771 1, count); 5772 } 5773 5774 } 5775 TEST_UTIL.deleteTable(tableName); 5776 } 5777 5778 @Test 5779 public void testEmptyFilterList() throws Exception { 5780 // Test Initialization. 5781 final TableName tableName = TableName.valueOf(name.getMethodName()); 5782 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) { 5783 5784 // Insert one row each region 5785 Put put = new Put(Bytes.toBytes("row")); 5786 put.addColumn(FAMILY, QUALIFIER, VALUE); 5787 table.put(put); 5788 5789 List<Result> scanResults = new LinkedList<>(); 5790 Scan scan = new Scan(); 5791 scan.setFilter(new FilterList()); 5792 try (ResultScanner scanner = table.getScanner(scan)) { 5793 for (Result r : scanner) { 5794 scanResults.add(r); 5795 } 5796 } 5797 assertEquals(1, scanResults.size()); 5798 Get g = new Get(Bytes.toBytes("row")); 5799 g.setFilter(new FilterList()); 5800 Result getResult = table.get(g); 5801 Result scanResult = scanResults.get(0); 5802 assertEquals(scanResult.rawCells().length, getResult.rawCells().length); 5803 for (int i = 0; i != scanResult.rawCells().length; ++i) { 5804 Cell scanCell = scanResult.rawCells()[i]; 5805 Cell getCell = getResult.rawCells()[i]; 5806 assertEquals(0, Bytes.compareTo(CellUtil.cloneRow(scanCell), 5807 CellUtil.cloneRow(getCell))); 5808 assertEquals(0, Bytes.compareTo(CellUtil.cloneFamily(scanCell), 5809 CellUtil.cloneFamily(getCell))); 5810 assertEquals(0, Bytes.compareTo(CellUtil.cloneQualifier(scanCell), 5811 CellUtil.cloneQualifier(getCell))); 5812 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(scanCell), 5813 CellUtil.cloneValue(getCell))); 5814 } 5815 } 5816 } 5817 5818 @Test 5819 public void testSmallScan() throws Exception { 5820 // Test Initialization. 5821 final TableName tableName = TableName.valueOf(name.getMethodName()); 5822 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) { 5823 5824 // Insert one row each region 5825 int insertNum = 10; 5826 for (int i = 0; i < 10; i++) { 5827 Put put = new Put(Bytes.toBytes("row" + String.format("%03d", i))); 5828 put.addColumn(FAMILY, QUALIFIER, VALUE); 5829 table.put(put); 5830 } 5831 5832 // normal scan 5833 try (ResultScanner scanner = table.getScanner(new Scan())) { 5834 int count = 0; 5835 for (Result r : scanner) { 5836 assertTrue(!r.isEmpty()); 5837 count++; 5838 } 5839 assertEquals(insertNum, count); 5840 } 5841 5842 // small scan 5843 Scan scan = new Scan(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); 5844 scan.setSmall(true); 5845 scan.setCaching(2); 5846 try (ResultScanner scanner = table.getScanner(scan)) { 5847 int count = 0; 5848 for (Result r : scanner) { 5849 assertTrue(!r.isEmpty()); 5850 count++; 5851 } 5852 assertEquals(insertNum, count); 5853 } 5854 } 5855 } 5856 5857 @Test 5858 public void testSuperSimpleWithReverseScan() throws Exception { 5859 final TableName tableName = TableName.valueOf(name.getMethodName()); 5860 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 5861 Put put = new Put(Bytes.toBytes("0-b11111-0000000000000000000")); 5862 put.addColumn(FAMILY, QUALIFIER, VALUE); 5863 ht.put(put); 5864 put = new Put(Bytes.toBytes("0-b11111-0000000000000000002")); 5865 put.addColumn(FAMILY, QUALIFIER, VALUE); 5866 ht.put(put); 5867 put = new Put(Bytes.toBytes("0-b11111-0000000000000000004")); 5868 put.addColumn(FAMILY, QUALIFIER, VALUE); 5869 ht.put(put); 5870 put = new Put(Bytes.toBytes("0-b11111-0000000000000000006")); 5871 put.addColumn(FAMILY, QUALIFIER, VALUE); 5872 ht.put(put); 5873 put = new Put(Bytes.toBytes("0-b11111-0000000000000000008")); 5874 put.addColumn(FAMILY, QUALIFIER, VALUE); 5875 ht.put(put); 5876 put = new Put(Bytes.toBytes("0-b22222-0000000000000000001")); 5877 put.addColumn(FAMILY, QUALIFIER, VALUE); 5878 ht.put(put); 5879 put = new Put(Bytes.toBytes("0-b22222-0000000000000000003")); 5880 put.addColumn(FAMILY, QUALIFIER, VALUE); 5881 ht.put(put); 5882 put = new Put(Bytes.toBytes("0-b22222-0000000000000000005")); 5883 put.addColumn(FAMILY, QUALIFIER, VALUE); 5884 ht.put(put); 5885 put = new Put(Bytes.toBytes("0-b22222-0000000000000000007")); 5886 put.addColumn(FAMILY, QUALIFIER, VALUE); 5887 ht.put(put); 5888 put = new Put(Bytes.toBytes("0-b22222-0000000000000000009")); 5889 put.addColumn(FAMILY, QUALIFIER, VALUE); 5890 ht.put(put); 5891 Scan scan = new Scan(Bytes.toBytes("0-b11111-9223372036854775807"), 5892 Bytes.toBytes("0-b11111-0000000000000000000")); 5893 scan.setReversed(true); 5894 try (ResultScanner scanner = ht.getScanner(scan)) { 5895 Result result = scanner.next(); 5896 assertTrue(Bytes.equals(result.getRow(), 5897 Bytes.toBytes("0-b11111-0000000000000000008"))); 5898 } 5899 } 5900 } 5901 5902 @Test 5903 public void testFiltersWithReverseScan() throws Exception { 5904 final TableName tableName = TableName.valueOf(name.getMethodName()); 5905 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 5906 byte[][] ROWS = makeN(ROW, 10); 5907 byte[][] QUALIFIERS = {Bytes.toBytes("col0-<d2v1>-<d3v2>"), 5908 Bytes.toBytes("col1-<d2v1>-<d3v2>"), 5909 Bytes.toBytes("col2-<d2v1>-<d3v2>"), 5910 Bytes.toBytes("col3-<d2v1>-<d3v2>"), 5911 Bytes.toBytes("col4-<d2v1>-<d3v2>"), 5912 Bytes.toBytes("col5-<d2v1>-<d3v2>"), 5913 Bytes.toBytes("col6-<d2v1>-<d3v2>"), 5914 Bytes.toBytes("col7-<d2v1>-<d3v2>"), 5915 Bytes.toBytes("col8-<d2v1>-<d3v2>"), 5916 Bytes.toBytes("col9-<d2v1>-<d3v2>")}; 5917 for (int i = 0; i < 10; i++) { 5918 Put put = new Put(ROWS[i]); 5919 put.addColumn(FAMILY, QUALIFIERS[i], VALUE); 5920 ht.put(put); 5921 } 5922 Scan scan = new Scan(); 5923 scan.setReversed(true); 5924 scan.addFamily(FAMILY); 5925 Filter filter = new QualifierFilter(CompareOperator.EQUAL, 5926 new RegexStringComparator("col[1-5]")); 5927 scan.setFilter(filter); 5928 try (ResultScanner scanner = ht.getScanner(scan)) { 5929 int expectedIndex = 5; 5930 for (Result result : scanner) { 5931 assertEquals(1, result.size()); 5932 Cell c = result.rawCells()[0]; 5933 assertTrue(Bytes.equals(c.getRowArray(), c.getRowOffset(), c.getRowLength(), 5934 ROWS[expectedIndex], 0, ROWS[expectedIndex].length)); 5935 assertTrue(Bytes.equals(c.getQualifierArray(), c.getQualifierOffset(), 5936 c.getQualifierLength(), QUALIFIERS[expectedIndex], 0, 5937 QUALIFIERS[expectedIndex].length)); 5938 expectedIndex--; 5939 } 5940 assertEquals(0, expectedIndex); 5941 } 5942 } 5943 } 5944 5945 @Test 5946 public void testKeyOnlyFilterWithReverseScan() throws Exception { 5947 final TableName tableName = TableName.valueOf(name.getMethodName()); 5948 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 5949 byte[][] ROWS = makeN(ROW, 10); 5950 byte[][] QUALIFIERS = {Bytes.toBytes("col0-<d2v1>-<d3v2>"), 5951 Bytes.toBytes("col1-<d2v1>-<d3v2>"), 5952 Bytes.toBytes("col2-<d2v1>-<d3v2>"), 5953 Bytes.toBytes("col3-<d2v1>-<d3v2>"), 5954 Bytes.toBytes("col4-<d2v1>-<d3v2>"), 5955 Bytes.toBytes("col5-<d2v1>-<d3v2>"), 5956 Bytes.toBytes("col6-<d2v1>-<d3v2>"), 5957 Bytes.toBytes("col7-<d2v1>-<d3v2>"), 5958 Bytes.toBytes("col8-<d2v1>-<d3v2>"), 5959 Bytes.toBytes("col9-<d2v1>-<d3v2>")}; 5960 for (int i = 0; i < 10; i++) { 5961 Put put = new Put(ROWS[i]); 5962 put.addColumn(FAMILY, QUALIFIERS[i], VALUE); 5963 ht.put(put); 5964 } 5965 Scan scan = new Scan(); 5966 scan.setReversed(true); 5967 scan.addFamily(FAMILY); 5968 Filter filter = new KeyOnlyFilter(true); 5969 scan.setFilter(filter); 5970 try (ResultScanner scanner = ht.getScanner(scan)) { 5971 int count = 0; 5972 for (Result result : ht.getScanner(scan)) { 5973 assertEquals(1, result.size()); 5974 assertEquals(Bytes.SIZEOF_INT, result.rawCells()[0].getValueLength()); 5975 assertEquals(VALUE.length, Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0]))); 5976 count++; 5977 } 5978 assertEquals(10, count); 5979 } 5980 } 5981 } 5982 5983 /** 5984 * Test simple table and non-existent row cases. 5985 */ 5986 @Test 5987 public void testSimpleMissingWithReverseScan() throws Exception { 5988 final TableName tableName = TableName.valueOf(name.getMethodName()); 5989 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 5990 byte[][] ROWS = makeN(ROW, 4); 5991 5992 // Try to get a row on an empty table 5993 Scan scan = new Scan(); 5994 scan.setReversed(true); 5995 Result result = getSingleScanResult(ht, scan); 5996 assertNullResult(result); 5997 5998 scan = new Scan(ROWS[0]); 5999 scan.setReversed(true); 6000 result = getSingleScanResult(ht, scan); 6001 assertNullResult(result); 6002 6003 scan = new Scan(ROWS[0], ROWS[1]); 6004 scan.setReversed(true); 6005 result = getSingleScanResult(ht, scan); 6006 assertNullResult(result); 6007 6008 scan = new Scan(); 6009 scan.setReversed(true); 6010 scan.addFamily(FAMILY); 6011 result = getSingleScanResult(ht, scan); 6012 assertNullResult(result); 6013 6014 scan = new Scan(); 6015 scan.setReversed(true); 6016 scan.addColumn(FAMILY, QUALIFIER); 6017 result = getSingleScanResult(ht, scan); 6018 assertNullResult(result); 6019 6020 // Insert a row 6021 6022 Put put = new Put(ROWS[2]); 6023 put.addColumn(FAMILY, QUALIFIER, VALUE); 6024 ht.put(put); 6025 6026 // Make sure we can scan the row 6027 scan = new Scan(); 6028 scan.setReversed(true); 6029 result = getSingleScanResult(ht, scan); 6030 assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE); 6031 6032 scan = new Scan(ROWS[3], ROWS[0]); 6033 scan.setReversed(true); 6034 result = getSingleScanResult(ht, scan); 6035 assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE); 6036 6037 scan = new Scan(ROWS[2], ROWS[1]); 6038 scan.setReversed(true); 6039 result = getSingleScanResult(ht, scan); 6040 assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE); 6041 6042 // Try to scan empty rows around it 6043 // Introduced MemStore#shouldSeekForReverseScan to fix the following 6044 scan = new Scan(ROWS[1]); 6045 scan.setReversed(true); 6046 result = getSingleScanResult(ht, scan); 6047 assertNullResult(result); 6048 } 6049 } 6050 6051 @Test 6052 public void testNullWithReverseScan() throws Exception { 6053 final TableName tableName = TableName.valueOf(name.getMethodName()); 6054 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 6055 // Null qualifier (should work) 6056 Put put = new Put(ROW); 6057 put.addColumn(FAMILY, null, VALUE); 6058 ht.put(put); 6059 scanTestNull(ht, ROW, FAMILY, VALUE, true); 6060 Delete delete = new Delete(ROW); 6061 delete.addColumns(FAMILY, null); 6062 ht.delete(delete); 6063 } 6064 6065 // Use a new table 6066 try (Table ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName() + "2"), FAMILY)) { 6067 // Empty qualifier, byte[0] instead of null (should work) 6068 Put put = new Put(ROW); 6069 put.addColumn(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE); 6070 ht.put(put); 6071 scanTestNull(ht, ROW, FAMILY, VALUE, true); 6072 TEST_UTIL.flush(); 6073 scanTestNull(ht, ROW, FAMILY, VALUE, true); 6074 Delete delete = new Delete(ROW); 6075 delete.addColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY); 6076 ht.delete(delete); 6077 // Null value 6078 put = new Put(ROW); 6079 put.addColumn(FAMILY, QUALIFIER, null); 6080 ht.put(put); 6081 Scan scan = new Scan(); 6082 scan.setReversed(true); 6083 scan.addColumn(FAMILY, QUALIFIER); 6084 Result result = getSingleScanResult(ht, scan); 6085 assertSingleResult(result, ROW, FAMILY, QUALIFIER, null); 6086 } 6087 } 6088 6089 @Test 6090 @SuppressWarnings("checkstyle:MethodLength") 6091 public void testDeletesWithReverseScan() throws Exception { 6092 final TableName tableName = TableName.valueOf(name.getMethodName()); 6093 byte[][] ROWS = makeNAscii(ROW, 6); 6094 byte[][] FAMILIES = makeNAscii(FAMILY, 3); 6095 byte[][] VALUES = makeN(VALUE, 5); 6096 long[] ts = { 1000, 2000, 3000, 4000, 5000 }; 6097 try (Table ht = TEST_UTIL.createTable(tableName, FAMILIES, 3)) { 6098 6099 Put put = new Put(ROW); 6100 put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); 6101 put.addColumn(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]); 6102 ht.put(put); 6103 6104 Delete delete = new Delete(ROW); 6105 delete.addFamily(FAMILIES[0], ts[0]); 6106 ht.delete(delete); 6107 6108 Scan scan = new Scan(ROW); 6109 scan.setReversed(true); 6110 scan.addFamily(FAMILIES[0]); 6111 scan.setMaxVersions(Integer.MAX_VALUE); 6112 Result result = getSingleScanResult(ht, scan); 6113 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[]{ts[1]}, 6114 new byte[][]{VALUES[1]}, 0, 0); 6115 6116 // Test delete latest version 6117 put = new Put(ROW); 6118 put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); 6119 put.addColumn(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]); 6120 put.addColumn(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]); 6121 put.addColumn(FAMILIES[0], null, ts[4], VALUES[4]); 6122 put.addColumn(FAMILIES[0], null, ts[2], VALUES[2]); 6123 put.addColumn(FAMILIES[0], null, ts[3], VALUES[3]); 6124 ht.put(put); 6125 6126 delete = new Delete(ROW); 6127 delete.addColumn(FAMILIES[0], QUALIFIER); // ts[4] 6128 ht.delete(delete); 6129 6130 scan = new Scan(ROW); 6131 scan.setReversed(true); 6132 scan.addColumn(FAMILIES[0], QUALIFIER); 6133 scan.setMaxVersions(Integer.MAX_VALUE); 6134 result = getSingleScanResult(ht, scan); 6135 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[]{ts[1], 6136 ts[2], ts[3]}, new byte[][]{VALUES[1], VALUES[2], VALUES[3]}, 0, 2); 6137 6138 // Test for HBASE-1847 6139 delete = new Delete(ROW); 6140 delete.addColumn(FAMILIES[0], null); 6141 ht.delete(delete); 6142 6143 // Cleanup null qualifier 6144 delete = new Delete(ROW); 6145 delete.addColumns(FAMILIES[0], null); 6146 ht.delete(delete); 6147 6148 // Expected client behavior might be that you can re-put deleted values 6149 // But alas, this is not to be. We can't put them back in either case. 6150 6151 put = new Put(ROW); 6152 put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); 6153 put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); 6154 ht.put(put); 6155 6156 // The Scanner returns the previous values, the expected-naive-unexpected 6157 // behavior 6158 6159 scan = new Scan(ROW); 6160 scan.setReversed(true); 6161 scan.addFamily(FAMILIES[0]); 6162 scan.setMaxVersions(Integer.MAX_VALUE); 6163 result = getSingleScanResult(ht, scan); 6164 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[]{ts[1], 6165 ts[2], ts[3]}, new byte[][]{VALUES[1], VALUES[2], VALUES[3]}, 0, 2); 6166 6167 // Test deleting an entire family from one row but not the other various 6168 // ways 6169 6170 put = new Put(ROWS[0]); 6171 put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]); 6172 put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]); 6173 put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]); 6174 put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]); 6175 ht.put(put); 6176 6177 put = new Put(ROWS[1]); 6178 put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]); 6179 put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]); 6180 put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]); 6181 put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]); 6182 ht.put(put); 6183 6184 put = new Put(ROWS[2]); 6185 put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]); 6186 put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]); 6187 put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]); 6188 put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]); 6189 ht.put(put); 6190 6191 delete = new Delete(ROWS[0]); 6192 delete.addFamily(FAMILIES[2]); 6193 ht.delete(delete); 6194 6195 delete = new Delete(ROWS[1]); 6196 delete.addColumns(FAMILIES[1], QUALIFIER); 6197 ht.delete(delete); 6198 6199 delete = new Delete(ROWS[2]); 6200 delete.addColumn(FAMILIES[1], QUALIFIER); 6201 delete.addColumn(FAMILIES[1], QUALIFIER); 6202 delete.addColumn(FAMILIES[2], QUALIFIER); 6203 ht.delete(delete); 6204 6205 scan = new Scan(ROWS[0]); 6206 scan.setReversed(true); 6207 scan.addFamily(FAMILIES[1]); 6208 scan.addFamily(FAMILIES[2]); 6209 scan.setMaxVersions(Integer.MAX_VALUE); 6210 result = getSingleScanResult(ht, scan); 6211 assertTrue("Expected 2 keys but received " + result.size(), 6212 result.size() == 2); 6213 assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, new long[]{ts[0], 6214 ts[1]}, new byte[][]{VALUES[0], VALUES[1]}, 0, 1); 6215 6216 scan = new Scan(ROWS[1]); 6217 scan.setReversed(true); 6218 scan.addFamily(FAMILIES[1]); 6219 scan.addFamily(FAMILIES[2]); 6220 scan.setMaxVersions(Integer.MAX_VALUE); 6221 result = getSingleScanResult(ht, scan); 6222 assertTrue("Expected 2 keys but received " + result.size(), 6223 result.size() == 2); 6224 6225 scan = new Scan(ROWS[2]); 6226 scan.setReversed(true); 6227 scan.addFamily(FAMILIES[1]); 6228 scan.addFamily(FAMILIES[2]); 6229 scan.setMaxVersions(Integer.MAX_VALUE); 6230 result = getSingleScanResult(ht, scan); 6231 assertEquals(1, result.size()); 6232 assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER, 6233 new long[]{ts[2]}, new byte[][]{VALUES[2]}, 0, 0); 6234 6235 // Test if we delete the family first in one row (HBASE-1541) 6236 6237 delete = new Delete(ROWS[3]); 6238 delete.addFamily(FAMILIES[1]); 6239 ht.delete(delete); 6240 6241 put = new Put(ROWS[3]); 6242 put.addColumn(FAMILIES[2], QUALIFIER, VALUES[0]); 6243 ht.put(put); 6244 6245 put = new Put(ROWS[4]); 6246 put.addColumn(FAMILIES[1], QUALIFIER, VALUES[1]); 6247 put.addColumn(FAMILIES[2], QUALIFIER, VALUES[2]); 6248 ht.put(put); 6249 6250 scan = new Scan(ROWS[4]); 6251 scan.setReversed(true); 6252 scan.addFamily(FAMILIES[1]); 6253 scan.addFamily(FAMILIES[2]); 6254 scan.setMaxVersions(Integer.MAX_VALUE); 6255 try (ResultScanner scanner = ht.getScanner(scan)) { 6256 result = scanner.next(); 6257 assertTrue("Expected 2 keys but received " + result.size(), 6258 result.size() == 2); 6259 assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[4])); 6260 assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[1]), ROWS[4])); 6261 assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[1])); 6262 assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[1]), VALUES[2])); 6263 result = scanner.next(); 6264 assertTrue("Expected 1 key but received " + result.size(), 6265 result.size() == 1); 6266 assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[3])); 6267 assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[0])); 6268 } 6269 } 6270 } 6271 6272 /** 6273 * Tests reversed scan under multi regions 6274 */ 6275 @Test 6276 public void testReversedScanUnderMultiRegions() throws Exception { 6277 // Test Initialization. 6278 final TableName tableName = TableName.valueOf(name.getMethodName()); 6279 byte[] maxByteArray = ConnectionUtils.MAX_BYTE_ARRAY; 6280 byte[][] splitRows = new byte[][] { Bytes.toBytes("005"), 6281 Bytes.add(Bytes.toBytes("005"), Bytes.multiple(maxByteArray, 16)), 6282 Bytes.toBytes("006"), 6283 Bytes.add(Bytes.toBytes("006"), Bytes.multiple(maxByteArray, 8)), 6284 Bytes.toBytes("007"), 6285 Bytes.add(Bytes.toBytes("007"), Bytes.multiple(maxByteArray, 4)), 6286 Bytes.toBytes("008"), Bytes.multiple(maxByteArray, 2) }; 6287 try (Table table = TEST_UTIL.createTable(tableName, FAMILY, splitRows)) { 6288 TEST_UTIL.waitUntilAllRegionsAssigned(table.getName()); 6289 6290 try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) { 6291 assertEquals(splitRows.length + 1, l.getAllRegionLocations().size()); 6292 } 6293 // Insert one row each region 6294 int insertNum = splitRows.length; 6295 for (int i = 0; i < insertNum; i++) { 6296 Put put = new Put(splitRows[i]); 6297 put.addColumn(FAMILY, QUALIFIER, VALUE); 6298 table.put(put); 6299 } 6300 6301 // scan forward 6302 try (ResultScanner scanner = table.getScanner(new Scan())) { 6303 int count = 0; 6304 for (Result r : scanner) { 6305 assertTrue(!r.isEmpty()); 6306 count++; 6307 } 6308 assertEquals(insertNum, count); 6309 } 6310 6311 // scan backward 6312 Scan scan = new Scan(); 6313 scan.setReversed(true); 6314 try (ResultScanner scanner = table.getScanner(scan)) { 6315 int count = 0; 6316 byte[] lastRow = null; 6317 for (Result r : scanner) { 6318 assertTrue(!r.isEmpty()); 6319 count++; 6320 byte[] thisRow = r.getRow(); 6321 if (lastRow != null) { 6322 assertTrue("Error scan order, last row= " + Bytes.toString(lastRow) 6323 + ",this row=" + Bytes.toString(thisRow), 6324 Bytes.compareTo(thisRow, lastRow) < 0); 6325 } 6326 lastRow = thisRow; 6327 } 6328 assertEquals(insertNum, count); 6329 } 6330 } 6331 } 6332 6333 /** 6334 * Tests reversed scan under multi regions 6335 */ 6336 @Test 6337 public void testSmallReversedScanUnderMultiRegions() throws Exception { 6338 // Test Initialization. 6339 final TableName tableName = TableName.valueOf(name.getMethodName()); 6340 byte[][] splitRows = new byte[][]{ 6341 Bytes.toBytes("000"), Bytes.toBytes("002"), Bytes.toBytes("004"), 6342 Bytes.toBytes("006"), Bytes.toBytes("008"), Bytes.toBytes("010")}; 6343 try (Table table = TEST_UTIL.createTable(tableName, FAMILY, splitRows)) { 6344 TEST_UTIL.waitUntilAllRegionsAssigned(table.getName()); 6345 6346 try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) { 6347 assertEquals(splitRows.length + 1, l.getAllRegionLocations().size()); 6348 } 6349 for (byte[] splitRow : splitRows) { 6350 Put put = new Put(splitRow); 6351 put.addColumn(FAMILY, QUALIFIER, VALUE); 6352 table.put(put); 6353 6354 byte[] nextRow = Bytes.copy(splitRow); 6355 nextRow[nextRow.length - 1]++; 6356 6357 put = new Put(nextRow); 6358 put.addColumn(FAMILY, QUALIFIER, VALUE); 6359 table.put(put); 6360 } 6361 6362 // scan forward 6363 try (ResultScanner scanner = table.getScanner(new Scan())) { 6364 int count = 0; 6365 for (Result r : scanner) { 6366 assertTrue(!r.isEmpty()); 6367 count++; 6368 } 6369 assertEquals(12, count); 6370 } 6371 6372 reverseScanTest(table, false); 6373 reverseScanTest(table, true); 6374 } 6375 } 6376 6377 private void reverseScanTest(Table table, boolean small) throws IOException { 6378 // scan backward 6379 Scan scan = new Scan(); 6380 scan.setReversed(true); 6381 try (ResultScanner scanner = table.getScanner(scan)) { 6382 int count = 0; 6383 byte[] lastRow = null; 6384 for (Result r : scanner) { 6385 assertTrue(!r.isEmpty()); 6386 count++; 6387 byte[] thisRow = r.getRow(); 6388 if (lastRow != null) { 6389 assertTrue("Error scan order, last row= " + Bytes.toString(lastRow) 6390 + ",this row=" + Bytes.toString(thisRow), 6391 Bytes.compareTo(thisRow, lastRow) < 0); 6392 } 6393 lastRow = thisRow; 6394 } 6395 assertEquals(12, count); 6396 } 6397 6398 scan = new Scan(); 6399 scan.setSmall(small); 6400 scan.setReversed(true); 6401 scan.setStartRow(Bytes.toBytes("002")); 6402 try (ResultScanner scanner = table.getScanner(scan)) { 6403 int count = 0; 6404 byte[] lastRow = null; 6405 for (Result r : scanner) { 6406 assertTrue(!r.isEmpty()); 6407 count++; 6408 byte[] thisRow = r.getRow(); 6409 if (lastRow != null) { 6410 assertTrue("Error scan order, last row= " + Bytes.toString(lastRow) 6411 + ",this row=" + Bytes.toString(thisRow), 6412 Bytes.compareTo(thisRow, lastRow) < 0); 6413 } 6414 lastRow = thisRow; 6415 } 6416 assertEquals(3, count); // 000 001 002 6417 } 6418 6419 scan = new Scan(); 6420 scan.setSmall(small); 6421 scan.setReversed(true); 6422 scan.setStartRow(Bytes.toBytes("002")); 6423 scan.setStopRow(Bytes.toBytes("000")); 6424 try (ResultScanner scanner = table.getScanner(scan)) { 6425 int count = 0; 6426 byte[] lastRow = null; 6427 for (Result r : scanner) { 6428 assertTrue(!r.isEmpty()); 6429 count++; 6430 byte[] thisRow = r.getRow(); 6431 if (lastRow != null) { 6432 assertTrue("Error scan order, last row= " + Bytes.toString(lastRow) 6433 + ",this row=" + Bytes.toString(thisRow), 6434 Bytes.compareTo(thisRow, lastRow) < 0); 6435 } 6436 lastRow = thisRow; 6437 } 6438 assertEquals(2, count); // 001 002 6439 } 6440 6441 scan = new Scan(); 6442 scan.setSmall(small); 6443 scan.setReversed(true); 6444 scan.setStartRow(Bytes.toBytes("001")); 6445 try (ResultScanner scanner = table.getScanner(scan)) { 6446 int count = 0; 6447 byte[] lastRow = null; 6448 for (Result r : scanner) { 6449 assertTrue(!r.isEmpty()); 6450 count++; 6451 byte[] thisRow = r.getRow(); 6452 if (lastRow != null) { 6453 assertTrue("Error scan order, last row= " + Bytes.toString(lastRow) 6454 + ",this row=" + Bytes.toString(thisRow), 6455 Bytes.compareTo(thisRow, lastRow) < 0); 6456 } 6457 lastRow = thisRow; 6458 } 6459 assertEquals(2, count); // 000 001 6460 } 6461 6462 scan = new Scan(); 6463 scan.setSmall(small); 6464 scan.setReversed(true); 6465 scan.setStartRow(Bytes.toBytes("000")); 6466 try (ResultScanner scanner = table.getScanner(scan)) { 6467 int count = 0; 6468 byte[] lastRow = null; 6469 for (Result r : scanner) { 6470 assertTrue(!r.isEmpty()); 6471 count++; 6472 byte[] thisRow = r.getRow(); 6473 if (lastRow != null) { 6474 assertTrue("Error scan order, last row= " + Bytes.toString(lastRow) 6475 + ",this row=" + Bytes.toString(thisRow), 6476 Bytes.compareTo(thisRow, lastRow) < 0); 6477 } 6478 lastRow = thisRow; 6479 } 6480 assertEquals(1, count); // 000 6481 } 6482 6483 scan = new Scan(); 6484 scan.setSmall(small); 6485 scan.setReversed(true); 6486 scan.setStartRow(Bytes.toBytes("006")); 6487 scan.setStopRow(Bytes.toBytes("002")); 6488 try (ResultScanner scanner = table.getScanner(scan)) { 6489 int count = 0; 6490 byte[] lastRow = null; 6491 for (Result r : scanner) { 6492 assertTrue(!r.isEmpty()); 6493 count++; 6494 byte[] thisRow = r.getRow(); 6495 if (lastRow != null) { 6496 assertTrue("Error scan order, last row= " + Bytes.toString(lastRow) 6497 + ",this row=" + Bytes.toString(thisRow), 6498 Bytes.compareTo(thisRow, lastRow) < 0); 6499 } 6500 lastRow = thisRow; 6501 } 6502 assertEquals(4, count); // 003 004 005 006 6503 } 6504 } 6505 6506 private static Pair<byte[][], byte[][]> getStartEndKeys(List<RegionLocations> regions) { 6507 final byte[][] startKeyList = new byte[regions.size()][]; 6508 final byte[][] endKeyList = new byte[regions.size()][]; 6509 6510 for (int i = 0; i < regions.size(); i++) { 6511 RegionInfo region = regions.get(i).getRegionLocation().getRegion(); 6512 startKeyList[i] = region.getStartKey(); 6513 endKeyList[i] = region.getEndKey(); 6514 } 6515 6516 return new Pair<>(startKeyList, endKeyList); 6517 } 6518 6519 @Test 6520 public void testGetStartEndKeysWithRegionReplicas() throws IOException { 6521 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); 6522 HColumnDescriptor fam = new HColumnDescriptor(FAMILY); 6523 htd.addFamily(fam); 6524 byte[][] KEYS = HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE; 6525 Admin admin = TEST_UTIL.getAdmin(); 6526 admin.createTable(htd, KEYS); 6527 List<HRegionInfo> regions = admin.getTableRegions(htd.getTableName()); 6528 6529 HRegionLocator locator = 6530 (HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName()); 6531 for (int regionReplication = 1; regionReplication < 4; regionReplication++) { 6532 List<RegionLocations> regionLocations = new ArrayList<>(); 6533 6534 // mock region locations coming from meta with multiple replicas 6535 for (HRegionInfo region : regions) { 6536 HRegionLocation[] arr = new HRegionLocation[regionReplication]; 6537 for (int i = 0; i < arr.length; i++) { 6538 arr[i] = new HRegionLocation(RegionReplicaUtil.getRegionInfoForReplica(region, i), null); 6539 } 6540 regionLocations.add(new RegionLocations(arr)); 6541 } 6542 6543 Pair<byte[][], byte[][]> startEndKeys = locator.getStartEndKeys(regionLocations); 6544 6545 assertEquals(KEYS.length + 1, startEndKeys.getFirst().length); 6546 6547 for (int i = 0; i < KEYS.length + 1; i++) { 6548 byte[] startKey = i == 0 ? HConstants.EMPTY_START_ROW : KEYS[i - 1]; 6549 byte[] endKey = i == KEYS.length ? HConstants.EMPTY_END_ROW : KEYS[i]; 6550 assertArrayEquals(startKey, startEndKeys.getFirst()[i]); 6551 assertArrayEquals(endKey, startEndKeys.getSecond()[i]); 6552 } 6553 } 6554 } 6555 6556 @Test 6557 public void testFilterAllRecords() throws IOException { 6558 Scan scan = new Scan(); 6559 scan.setBatch(1); 6560 scan.setCaching(1); 6561 // Filter out any records 6562 scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new InclusiveStopFilter(new byte[0]))); 6563 try (Table table = TEST_UTIL.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME)) { 6564 try (ResultScanner s = table.getScanner(scan)) { 6565 assertNull(s.next()); 6566 } 6567 } 6568 } 6569 6570 @Test 6571 public void testRegionCache() throws IOException { 6572 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); 6573 HColumnDescriptor fam = new HColumnDescriptor(FAMILY); 6574 htd.addFamily(fam); 6575 byte[][] KEYS = HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE; 6576 try (Admin admin = TEST_UTIL.getAdmin()) { 6577 admin.createTable(htd, KEYS); 6578 HRegionLocator locator = 6579 (HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName()); 6580 List<HRegionLocation> results = locator.getAllRegionLocations(); 6581 int number = ((ConnectionImplementation) admin.getConnection()) 6582 .getNumberOfCachedRegionLocations(htd.getTableName()); 6583 assertEquals(results.size(), number); 6584 ConnectionImplementation conn = ((ConnectionImplementation) admin.getConnection()); 6585 assertNotNull("Can't get cached location for row aaa", 6586 conn.getCachedLocation(htd.getTableName(), Bytes.toBytes("aaa"))); 6587 for (byte[] startKey : HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE) { 6588 assertNotNull("Can't get cached location for row " + 6589 Bytes.toString(startKey), (conn.getCachedLocation(htd.getTableName(), startKey))); 6590 } 6591 } 6592 } 6593 6594 @Test 6595 public void testCellSizeLimit() throws IOException { 6596 final TableName tableName = TableName.valueOf("testCellSizeLimit"); 6597 HTableDescriptor htd = new HTableDescriptor(tableName); 6598 htd.setConfiguration(HRegion.HBASE_MAX_CELL_SIZE_KEY, Integer.toString(10 * 1024)); // 10K 6599 HColumnDescriptor fam = new HColumnDescriptor(FAMILY); 6600 htd.addFamily(fam); 6601 try (Admin admin = TEST_UTIL.getAdmin()) { 6602 admin.createTable(htd); 6603 } 6604 // Will succeed 6605 try (Table t = TEST_UTIL.getConnection().getTable(tableName)) { 6606 t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, Bytes.toBytes(0L))); 6607 t.increment(new Increment(ROW).addColumn(FAMILY, QUALIFIER, 1L)); 6608 } 6609 // Will succeed 6610 try (Table t = TEST_UTIL.getConnection().getTable(tableName)) { 6611 t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, new byte[9*1024])); 6612 } 6613 // Will fail 6614 try (Table t = TEST_UTIL.getConnection().getTable(tableName)) { 6615 try { 6616 t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, new byte[10 * 1024])); 6617 fail("Oversize cell failed to trigger exception"); 6618 } catch (IOException e) { 6619 // expected 6620 } 6621 try { 6622 t.append(new Append(ROW).addColumn(FAMILY, QUALIFIER, new byte[2 * 1024])); 6623 fail("Oversize cell failed to trigger exception"); 6624 } catch (IOException e) { 6625 // expected 6626 } 6627 } 6628 } 6629 6630 @Test 6631 public void testDeleteSpecifiedVersionOfSpecifiedColumn() throws Exception { 6632 try (Admin admin = TEST_UTIL.getAdmin()) { 6633 final TableName tableName = TableName.valueOf(name.getMethodName()); 6634 6635 byte[][] VALUES = makeN(VALUE, 5); 6636 long[] ts = {1000, 2000, 3000, 4000, 5000}; 6637 6638 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5)) { 6639 6640 Put put = new Put(ROW); 6641 // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER 6642 for (int t = 0; t < 4; t++) { 6643 put.addColumn(FAMILY, QUALIFIER, ts[t], VALUES[t]); 6644 } 6645 ht.put(put); 6646 6647 Delete delete = new Delete(ROW); 6648 // Delete version 3000 of column FAMILY:QUALIFIER 6649 delete.addColumn(FAMILY, QUALIFIER, ts[2]); 6650 ht.delete(delete); 6651 6652 Get get = new Get(ROW); 6653 get.addColumn(FAMILY, QUALIFIER); 6654 get.setMaxVersions(Integer.MAX_VALUE); 6655 Result result = ht.get(get); 6656 // verify version 1000,2000,4000 remains for column FAMILY:QUALIFIER 6657 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[1], ts[3]}, new byte[][]{ 6658 VALUES[0], VALUES[1], VALUES[3]}, 0, 2); 6659 6660 delete = new Delete(ROW); 6661 // Delete a version 5000 of column FAMILY:QUALIFIER which didn't exist 6662 delete.addColumn(FAMILY, QUALIFIER, ts[4]); 6663 ht.delete(delete); 6664 6665 get = new Get(ROW); 6666 get.addColumn(FAMILY, QUALIFIER); 6667 get.setMaxVersions(Integer.MAX_VALUE); 6668 result = ht.get(get); 6669 // verify version 1000,2000,4000 remains for column FAMILY:QUALIFIER 6670 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[1], ts[3]}, new byte[][]{ 6671 VALUES[0], VALUES[1], VALUES[3]}, 0, 2); 6672 } 6673 } 6674 } 6675 6676 @Test 6677 public void testDeleteLatestVersionOfSpecifiedColumn() throws Exception { 6678 try (Admin admin = TEST_UTIL.getAdmin()) { 6679 final TableName tableName = TableName.valueOf(name.getMethodName()); 6680 6681 byte[][] VALUES = makeN(VALUE, 5); 6682 long[] ts = {1000, 2000, 3000, 4000, 5000}; 6683 6684 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5)) { 6685 6686 Put put = new Put(ROW); 6687 // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER 6688 for (int t = 0; t < 4; t++) { 6689 put.addColumn(FAMILY, QUALIFIER, ts[t], VALUES[t]); 6690 } 6691 ht.put(put); 6692 6693 Delete delete = new Delete(ROW); 6694 // Delete latest version of column FAMILY:QUALIFIER 6695 delete.addColumn(FAMILY, QUALIFIER); 6696 ht.delete(delete); 6697 6698 Get get = new Get(ROW); 6699 get.addColumn(FAMILY, QUALIFIER); 6700 get.setMaxVersions(Integer.MAX_VALUE); 6701 Result result = ht.get(get); 6702 // verify version 1000,2000,3000 remains for column FAMILY:QUALIFIER 6703 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[1], ts[2]}, new byte[][]{ 6704 VALUES[0], VALUES[1], VALUES[2]}, 0, 2); 6705 6706 delete = new Delete(ROW); 6707 // Delete two latest version of column FAMILY:QUALIFIER 6708 delete.addColumn(FAMILY, QUALIFIER); 6709 delete.addColumn(FAMILY, QUALIFIER); 6710 ht.delete(delete); 6711 6712 get = new Get(ROW); 6713 get.addColumn(FAMILY, QUALIFIER); 6714 get.setMaxVersions(Integer.MAX_VALUE); 6715 result = ht.get(get); 6716 // verify version 1000 remains for column FAMILY:QUALIFIER 6717 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0]}, new byte[][]{VALUES[0]}, 6718 0, 0); 6719 6720 put = new Put(ROW); 6721 // Put a version 5000 of column FAMILY:QUALIFIER 6722 put.addColumn(FAMILY, QUALIFIER, ts[4], VALUES[4]); 6723 ht.put(put); 6724 6725 get = new Get(ROW); 6726 get.addColumn(FAMILY, QUALIFIER); 6727 get.setMaxVersions(Integer.MAX_VALUE); 6728 result = ht.get(get); 6729 // verify version 1000,5000 remains for column FAMILY:QUALIFIER 6730 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[4]}, new byte[][]{ 6731 VALUES[0], VALUES[4]}, 0, 1); 6732 } 6733 } 6734 } 6735 6736 /** 6737 * Test for HBASE-17125 6738 */ 6739 @Test 6740 public void testReadWithFilter() throws Exception { 6741 try (Admin admin = TEST_UTIL.getAdmin()) { 6742 final TableName tableName = TableName.valueOf(name.getMethodName()); 6743 try (Table table = TEST_UTIL.createTable(tableName, FAMILY, 3)) { 6744 6745 byte[] VALUEA = Bytes.toBytes("value-a"); 6746 byte[] VALUEB = Bytes.toBytes("value-b"); 6747 long[] ts = {1000, 2000, 3000, 4000}; 6748 6749 Put put = new Put(ROW); 6750 // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER 6751 for (int t = 0; t <= 3; t++) { 6752 if (t <= 1) { 6753 put.addColumn(FAMILY, QUALIFIER, ts[t], VALUEA); 6754 } else { 6755 put.addColumn(FAMILY, QUALIFIER, ts[t], VALUEB); 6756 } 6757 } 6758 table.put(put); 6759 6760 Scan scan = 6761 new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL, 6762 new SubstringComparator("value-a"))) 6763 .setMaxVersions(3); 6764 ResultScanner scanner = table.getScanner(scan); 6765 Result result = scanner.next(); 6766 // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3 6767 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0, 6768 0); 6769 6770 Get get = 6771 new Get(ROW) 6772 .setFilter(new ValueFilter(CompareOperator.EQUAL, 6773 new SubstringComparator("value-a"))) 6774 .setMaxVersions(3); 6775 result = table.get(get); 6776 // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3 6777 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0, 6778 0); 6779 6780 // Test with max versions 1, it should still read ts[1] 6781 scan = 6782 new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL, 6783 new SubstringComparator("value-a"))) 6784 .setMaxVersions(1); 6785 scanner = table.getScanner(scan); 6786 result = scanner.next(); 6787 // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3 6788 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0, 6789 0); 6790 6791 // Test with max versions 1, it should still read ts[1] 6792 get = 6793 new Get(ROW) 6794 .setFilter(new ValueFilter(CompareOperator.EQUAL, 6795 new SubstringComparator("value-a"))) 6796 .setMaxVersions(1); 6797 result = table.get(get); 6798 // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3 6799 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0, 6800 0); 6801 6802 // Test with max versions 5, it should still read ts[1] 6803 scan = 6804 new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL, 6805 new SubstringComparator("value-a"))) 6806 .setMaxVersions(5); 6807 scanner = table.getScanner(scan); 6808 result = scanner.next(); 6809 // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3 6810 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0, 6811 0); 6812 6813 // Test with max versions 5, it should still read ts[1] 6814 get = 6815 new Get(ROW) 6816 .setFilter(new ValueFilter(CompareOperator.EQUAL, 6817 new SubstringComparator("value-a"))) 6818 .setMaxVersions(5); 6819 result = table.get(get); 6820 // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3 6821 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0, 6822 0); 6823 } 6824 } 6825 } 6826 6827 @Test 6828 public void testCellUtilTypeMethods() throws IOException { 6829 final TableName tableName = TableName.valueOf(name.getMethodName()); 6830 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) { 6831 6832 final byte[] row = Bytes.toBytes("p"); 6833 Put p = new Put(row); 6834 p.addColumn(FAMILY, QUALIFIER, VALUE); 6835 table.put(p); 6836 6837 try (ResultScanner scanner = table.getScanner(new Scan())) { 6838 Result result = scanner.next(); 6839 assertNotNull(result); 6840 CellScanner cs = result.cellScanner(); 6841 assertTrue(cs.advance()); 6842 Cell c = cs.current(); 6843 assertTrue(CellUtil.isPut(c)); 6844 assertFalse(CellUtil.isDelete(c)); 6845 assertFalse(cs.advance()); 6846 assertNull(scanner.next()); 6847 } 6848 6849 Delete d = new Delete(row); 6850 d.addColumn(FAMILY, QUALIFIER); 6851 table.delete(d); 6852 6853 Scan scan = new Scan(); 6854 scan.setRaw(true); 6855 try (ResultScanner scanner = table.getScanner(scan)) { 6856 Result result = scanner.next(); 6857 assertNotNull(result); 6858 CellScanner cs = result.cellScanner(); 6859 assertTrue(cs.advance()); 6860 6861 // First cell should be the delete (masking the Put) 6862 Cell c = cs.current(); 6863 assertTrue("Cell should be a Delete: " + c, CellUtil.isDelete(c)); 6864 assertFalse("Cell should not be a Put: " + c, CellUtil.isPut(c)); 6865 6866 // Second cell should be the original Put 6867 assertTrue(cs.advance()); 6868 c = cs.current(); 6869 assertFalse("Cell should not be a Delete: " + c, CellUtil.isDelete(c)); 6870 assertTrue("Cell should be a Put: " + c, CellUtil.isPut(c)); 6871 6872 // No more cells in this row 6873 assertFalse(cs.advance()); 6874 6875 // No more results in this scan 6876 assertNull(scanner.next()); 6877 } 6878 } 6879 } 6880 6881 @Test(expected = DoNotRetryIOException.class) 6882 public void testCreateTableWithZeroRegionReplicas() throws Exception { 6883 TableName tableName = TableName.valueOf(name.getMethodName()); 6884 TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) 6885 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf"))) 6886 .setRegionReplication(0) 6887 .build(); 6888 6889 TEST_UTIL.getAdmin().createTable(desc); 6890 } 6891 6892 @Test(expected = DoNotRetryIOException.class) 6893 public void testModifyTableWithZeroRegionReplicas() throws Exception { 6894 TableName tableName = TableName.valueOf(name.getMethodName()); 6895 TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) 6896 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf"))) 6897 .build(); 6898 6899 TEST_UTIL.getAdmin().createTable(desc); 6900 TableDescriptor newDesc = TableDescriptorBuilder.newBuilder(desc) 6901 .setRegionReplication(0) 6902 .build(); 6903 6904 TEST_UTIL.getAdmin().modifyTable(newDesc); 6905 } 6906 6907 @Test(timeout = 60000) 6908 public void testModifyTableWithMemstoreData() throws Exception { 6909 TableName tableName = TableName.valueOf(name.getMethodName()); 6910 createTableAndValidateTableSchemaModification(tableName, true); 6911 } 6912 6913 @Test(timeout = 60000) 6914 public void testDeleteCFWithMemstoreData() throws Exception { 6915 TableName tableName = TableName.valueOf(name.getMethodName()); 6916 createTableAndValidateTableSchemaModification(tableName, false); 6917 } 6918 6919 /** 6920 * Create table and validate online schema modification 6921 * @param tableName Table name 6922 * @param modifyTable Modify table if true otherwise delete column family 6923 * @throws IOException in case of failures 6924 */ 6925 private void createTableAndValidateTableSchemaModification(TableName tableName, 6926 boolean modifyTable) throws Exception { 6927 Admin admin = TEST_UTIL.getAdmin(); 6928 // Create table with two Cfs 6929 byte[] cf1 = Bytes.toBytes("cf1"); 6930 byte[] cf2 = Bytes.toBytes("cf2"); 6931 TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName) 6932 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf1)) 6933 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf2)).build(); 6934 admin.createTable(tableDesc); 6935 6936 Table t = TEST_UTIL.getConnection().getTable(tableName); 6937 // Insert few records and flush the table 6938 t.put(new Put(ROW).addColumn(cf1, QUALIFIER, Bytes.toBytes("val1"))); 6939 t.put(new Put(ROW).addColumn(cf2, QUALIFIER, Bytes.toBytes("val2"))); 6940 admin.flush(tableName); 6941 Path tableDir = FSUtils.getTableDir(TEST_UTIL.getDefaultRootDirPath(), tableName); 6942 List<Path> regionDirs = FSUtils.getRegionDirs(TEST_UTIL.getTestFileSystem(), tableDir); 6943 assertTrue(regionDirs.size() == 1); 6944 List<Path> familyDirs = FSUtils.getFamilyDirs(TEST_UTIL.getTestFileSystem(), regionDirs.get(0)); 6945 assertTrue(familyDirs.size() == 2); 6946 6947 // Insert record but dont flush the table 6948 t.put(new Put(ROW).addColumn(cf1, QUALIFIER, Bytes.toBytes("val2"))); 6949 t.put(new Put(ROW).addColumn(cf2, QUALIFIER, Bytes.toBytes("val2"))); 6950 6951 if (modifyTable) { 6952 tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).removeColumnFamily(cf2).build(); 6953 admin.modifyTable(tableDesc); 6954 } else { 6955 admin.deleteColumnFamily(tableName, cf2); 6956 } 6957 // After table modification or delete family there should be only one CF in FS 6958 familyDirs = FSUtils.getFamilyDirs(TEST_UTIL.getTestFileSystem(), regionDirs.get(0)); 6959 assertTrue("CF dir count should be 1, but was " + familyDirs.size(), familyDirs.size() == 1); 6960 } 6961}