001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.client; 019 020import static org.hamcrest.CoreMatchers.instanceOf; 021import static org.junit.Assert.assertEquals; 022import static org.junit.Assert.assertFalse; 023import static org.junit.Assert.assertNotNull; 024import static org.junit.Assert.assertThat; 025import static org.junit.Assert.assertTrue; 026import static org.junit.Assert.fail; 027import java.io.IOException; 028import java.util.ArrayList; 029import java.util.Arrays; 030import java.util.Collection; 031import java.util.EnumSet; 032import java.util.HashSet; 033import java.util.List; 034import java.util.NavigableMap; 035import org.apache.hadoop.conf.Configuration; 036import org.apache.hadoop.hbase.Cell; 037import org.apache.hadoop.hbase.CellUtil; 038import org.apache.hadoop.hbase.ClusterMetrics.Option; 039import org.apache.hadoop.hbase.HBaseClassTestRule; 040import org.apache.hadoop.hbase.HColumnDescriptor; 041import org.apache.hadoop.hbase.HConstants; 042import org.apache.hadoop.hbase.HTableDescriptor; 043import org.apache.hadoop.hbase.KeyValue; 044import org.apache.hadoop.hbase.MiniHBaseCluster; 045import org.apache.hadoop.hbase.TableName; 046import org.apache.hadoop.hbase.TableNameTestRule; 047import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint; 048import org.apache.hadoop.hbase.master.LoadBalancer; 049import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; 050import org.apache.hadoop.hbase.testclassification.ClientTests; 051import org.apache.hadoop.hbase.testclassification.LargeTests; 052import org.apache.hadoop.hbase.util.Bytes; 053import org.junit.AfterClass; 054import org.junit.Assume; 055import org.junit.ClassRule; 056import org.junit.Rule; 057import org.junit.Test; 058import org.junit.experimental.categories.Category; 059import org.junit.runner.RunWith; 060import org.junit.runners.Parameterized; 061import org.slf4j.Logger; 062import org.slf4j.LoggerFactory; 063 064/** 065 * Run tests that use the HBase clients; {@link Table}. 066 * Sets up the HBase mini cluster once at start and runs through all client tests. 067 * Each creates a table named for the method and does its stuff against that. 068 * 069 * Parameterized to run with different registry implementations. 070 */ 071@Category({LargeTests.class, ClientTests.class}) 072@SuppressWarnings ("deprecation") 073@RunWith(Parameterized.class) 074public class TestFromClientSide4 extends FromClientSideBase { 075 private static final Logger LOG = LoggerFactory.getLogger(TestFromClientSide4.class); 076 @ClassRule 077 public static final HBaseClassTestRule CLASS_RULE = 078 HBaseClassTestRule.forClass(TestFromClientSide4.class); 079 @Rule 080 public TableNameTestRule name = new TableNameTestRule(); 081 082 // To keep the child classes happy. 083 TestFromClientSide4() { 084 } 085 086 public TestFromClientSide4(Class registry, int numHedgedReqs) throws Exception { 087 initialize(registry, numHedgedReqs, MultiRowMutationEndpoint.class); 088 } 089 090 @Parameterized.Parameters 091 public static Collection parameters() { 092 return Arrays.asList(new Object[][] { { MasterRegistry.class, 1 }, { MasterRegistry.class, 2 }, 093 { ZKConnectionRegistry.class, 1 } }); 094 } 095 096 @AfterClass 097 public static void tearDownAfterClass() throws Exception { 098 afterClass(); 099 } 100 101 /** 102 * Test batch operations with combination of valid and invalid args 103 */ 104 @Test public void testBatchOperationsWithErrors() throws Exception { 105 final TableName tableName = name.getTableName(); 106 try (Table foo = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 10)) { 107 108 int NUM_OPS = 100; 109 110 // 1.1 Put with no column families (local validation, runtime exception) 111 List<Put> puts = new ArrayList<>(NUM_OPS); 112 for (int i = 0; i != NUM_OPS; i++) { 113 Put put = new Put(Bytes.toBytes(i)); 114 puts.add(put); 115 } 116 117 try { 118 foo.put(puts); 119 fail(); 120 } catch (IllegalArgumentException e) { 121 // expected 122 assertEquals(NUM_OPS, puts.size()); 123 } 124 125 // 1.2 Put with invalid column family 126 puts.clear(); 127 for (int i = 0; i < NUM_OPS; i++) { 128 Put put = new Put(Bytes.toBytes(i)); 129 put.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY, Bytes.toBytes(i)); 130 puts.add(put); 131 } 132 133 try { 134 foo.put(puts); 135 fail(); 136 } catch (RetriesExhaustedException e) { 137 if (e instanceof RetriesExhaustedWithDetailsException) { 138 assertThat(((RetriesExhaustedWithDetailsException)e).exceptions.get(0), 139 instanceOf(NoSuchColumnFamilyException.class)); 140 } else { 141 assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class)); 142 } 143 } 144 145 // 2.1 Get non-existent rows 146 List<Get> gets = new ArrayList<>(NUM_OPS); 147 for (int i = 0; i < NUM_OPS; i++) { 148 Get get = new Get(Bytes.toBytes(i)); 149 gets.add(get); 150 } 151 Result[] getsResult = foo.get(gets); 152 assertNotNull(getsResult); 153 assertEquals(NUM_OPS, getsResult.length); 154 for (int i = 0; i < NUM_OPS; i++) { 155 Result getResult = getsResult[i]; 156 if (i % 2 == 0) { 157 assertFalse(getResult.isEmpty()); 158 } else { 159 assertTrue(getResult.isEmpty()); 160 } 161 } 162 163 // 2.2 Get with invalid column family 164 gets.clear(); 165 for (int i = 0; i < NUM_OPS; i++) { 166 Get get = new Get(Bytes.toBytes(i)); 167 get.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY); 168 gets.add(get); 169 } 170 try { 171 foo.get(gets); 172 fail(); 173 } catch (RetriesExhaustedException e) { 174 if (e instanceof RetriesExhaustedWithDetailsException) { 175 assertThat(((RetriesExhaustedWithDetailsException)e).exceptions.get(0), 176 instanceOf(NoSuchColumnFamilyException.class)); 177 } else { 178 assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class)); 179 } 180 } 181 182 // 3.1 Delete with invalid column family 183 List<Delete> deletes = new ArrayList<>(NUM_OPS); 184 for (int i = 0; i < NUM_OPS; i++) { 185 Delete delete = new Delete(Bytes.toBytes(i)); 186 delete.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY); 187 deletes.add(delete); 188 } 189 try { 190 foo.delete(deletes); 191 fail(); 192 } catch (RetriesExhaustedException e) { 193 if (e instanceof RetriesExhaustedWithDetailsException) { 194 assertThat(((RetriesExhaustedWithDetailsException)e).exceptions.get(0), 195 instanceOf(NoSuchColumnFamilyException.class)); 196 } else { 197 assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class)); 198 } 199 } 200 201 // all valid rows should have been deleted 202 gets.clear(); 203 for (int i = 0; i < NUM_OPS; i++) { 204 Get get = new Get(Bytes.toBytes(i)); 205 gets.add(get); 206 } 207 getsResult = foo.get(gets); 208 assertNotNull(getsResult); 209 assertEquals(NUM_OPS, getsResult.length); 210 for (Result getResult : getsResult) { 211 assertTrue(getResult.isEmpty()); 212 } 213 214 // 3.2 Delete non-existent rows 215 deletes.clear(); 216 for (int i = 0; i < NUM_OPS; i++) { 217 Delete delete = new Delete(Bytes.toBytes(i)); 218 deletes.add(delete); 219 } 220 foo.delete(deletes); 221 } 222 } 223 224 // 225 // JIRA Testers 226 // 227 228 /** 229 * HBASE-867 230 * If millions of columns in a column family, hbase scanner won't come up 231 * Test will create numRows rows, each with numColsPerRow columns 232 * (1 version each), and attempt to scan them all. 233 * To test at scale, up numColsPerRow to the millions 234 * (have not gotten that to work running as junit though) 235 */ 236 @Test public void testJiraTest867() throws Exception { 237 int numRows = 10; 238 int numColsPerRow = 2000; 239 240 final TableName tableName = name.getTableName(); 241 242 byte[][] ROWS = makeN(ROW, numRows); 243 byte[][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow); 244 245 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) { 246 247 // Insert rows 248 249 for (int i = 0; i < numRows; i++) { 250 Put put = new Put(ROWS[i]); 251 put.setDurability(Durability.SKIP_WAL); 252 for (int j = 0; j < numColsPerRow; j++) { 253 put.addColumn(FAMILY, QUALIFIERS[j], QUALIFIERS[j]); 254 } 255 assertEquals( 256 "Put expected to contain " + numColsPerRow + " columns but " + "only contains " + put 257 .size(), put.size(), numColsPerRow); 258 ht.put(put); 259 } 260 261 // Get a row 262 Get get = new Get(ROWS[numRows - 1]); 263 Result result = ht.get(get); 264 assertNumKeys(result, numColsPerRow); 265 Cell[] keys = result.rawCells(); 266 for (int i = 0; i < result.size(); i++) { 267 assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]); 268 } 269 270 // Scan the rows 271 Scan scan = new Scan(); 272 try (ResultScanner scanner = ht.getScanner(scan)) { 273 int rowCount = 0; 274 while ((result = scanner.next()) != null) { 275 assertNumKeys(result, numColsPerRow); 276 Cell[] kvs = result.rawCells(); 277 for (int i = 0; i < numColsPerRow; i++) { 278 assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]); 279 } 280 rowCount++; 281 } 282 assertEquals( 283 "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows", 284 rowCount, numRows); 285 } 286 287 // flush and try again 288 289 TEST_UTIL.flush(); 290 291 // Get a row 292 get = new Get(ROWS[numRows - 1]); 293 result = ht.get(get); 294 assertNumKeys(result, numColsPerRow); 295 keys = result.rawCells(); 296 for (int i = 0; i < result.size(); i++) { 297 assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]); 298 } 299 300 // Scan the rows 301 scan = new Scan(); 302 try (ResultScanner scanner = ht.getScanner(scan)) { 303 int rowCount = 0; 304 while ((result = scanner.next()) != null) { 305 assertNumKeys(result, numColsPerRow); 306 Cell[] kvs = result.rawCells(); 307 for (int i = 0; i < numColsPerRow; i++) { 308 assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]); 309 } 310 rowCount++; 311 } 312 assertEquals( 313 "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows", 314 rowCount, numRows); 315 } 316 } 317 } 318 319 /** 320 * HBASE-861 321 * get with timestamp will return a value if there is a version with an 322 * earlier timestamp 323 */ 324 @Test public void testJiraTest861() throws Exception { 325 final TableName tableName = name.getTableName(); 326 byte[][] VALUES = makeNAscii(VALUE, 7); 327 long[] STAMPS = makeStamps(7); 328 329 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) { 330 331 // Insert three versions 332 333 Put put = new Put(ROW); 334 put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); 335 put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 336 put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 337 ht.put(put); 338 339 // Get the middle value 340 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 341 342 // Try to get one version before (expect fail) 343 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]); 344 345 // Try to get one version after (expect fail) 346 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]); 347 348 // Try same from storefile 349 TEST_UTIL.flush(); 350 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 351 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]); 352 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]); 353 354 // Insert two more versions surrounding others, into memstore 355 put = new Put(ROW); 356 put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]); 357 put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]); 358 ht.put(put); 359 360 // Check we can get everything we should and can't get what we shouldn't 361 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]); 362 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]); 363 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 364 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); 365 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 366 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]); 367 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]); 368 369 // Try same from two storefiles 370 TEST_UTIL.flush(); 371 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]); 372 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]); 373 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 374 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); 375 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 376 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]); 377 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]); 378 } 379 } 380 381 /** 382 * HBASE-33 383 * Add a HTable get/obtainScanner method that retrieves all versions of a 384 * particular column and row between two timestamps 385 */ 386 @Test public void testJiraTest33() throws Exception { 387 final TableName tableName = name.getTableName(); 388 byte[][] VALUES = makeNAscii(VALUE, 7); 389 long[] STAMPS = makeStamps(7); 390 391 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) { 392 393 // Insert lots versions 394 395 Put put = new Put(ROW); 396 put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]); 397 put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 398 put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 399 put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); 400 put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 401 put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 402 ht.put(put); 403 404 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 405 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2); 406 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 407 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3); 408 409 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 410 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2); 411 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 412 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3); 413 414 // Try same from storefile 415 TEST_UTIL.flush(); 416 417 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 418 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2); 419 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 420 getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3); 421 422 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 423 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2); 424 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 425 scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3); 426 } 427 } 428 429 /** 430 * HBASE-1014 431 * commit(BatchUpdate) method should return timestamp 432 */ 433 @Test public void testJiraTest1014() throws Exception { 434 final TableName tableName = name.getTableName(); 435 436 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) { 437 438 long manualStamp = 12345; 439 440 // Insert lots versions 441 442 Put put = new Put(ROW); 443 put.addColumn(FAMILY, QUALIFIER, manualStamp, VALUE); 444 ht.put(put); 445 446 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE); 447 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp - 1); 448 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp + 1); 449 } 450 } 451 452 /** 453 * HBASE-1182 454 * Scan for columns > some timestamp 455 */ 456 @Test public void testJiraTest1182() throws Exception { 457 final TableName tableName = name.getTableName(); 458 byte[][] VALUES = makeNAscii(VALUE, 7); 459 long[] STAMPS = makeStamps(7); 460 461 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) { 462 463 // Insert lots versions 464 465 Put put = new Put(ROW); 466 put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]); 467 put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 468 put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 469 put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); 470 put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 471 put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 472 ht.put(put); 473 474 getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 475 getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5); 476 getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 477 478 scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 479 scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5); 480 scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 481 482 // Try same from storefile 483 TEST_UTIL.flush(); 484 485 getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 486 getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5); 487 getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 488 489 scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 490 scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5); 491 scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5); 492 } 493 } 494 495 /** 496 * HBASE-52 497 * Add a means of scanning over all versions 498 */ 499 @Test public void testJiraTest52() throws Exception { 500 final TableName tableName = name.getTableName(); 501 byte[][] VALUES = makeNAscii(VALUE, 7); 502 long[] STAMPS = makeStamps(7); 503 504 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) { 505 506 // Insert lots versions 507 508 Put put = new Put(ROW); 509 put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]); 510 put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 511 put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 512 put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); 513 put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 514 put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 515 ht.put(put); 516 517 getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 518 519 scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 520 521 // Try same from storefile 522 TEST_UTIL.flush(); 523 524 getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 525 526 scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5); 527 } 528 } 529 530 @Test 531 @SuppressWarnings("checkstyle:MethodLength") 532 public void testDuplicateVersions() throws Exception { 533 final TableName tableName = name.getTableName(); 534 535 long[] STAMPS = makeStamps(20); 536 byte[][] VALUES = makeNAscii(VALUE, 20); 537 538 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) { 539 540 // Insert 4 versions of same column 541 Put put = new Put(ROW); 542 put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 543 put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 544 put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 545 put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 546 ht.put(put); 547 548 // Verify we can get each one properly 549 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 550 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 551 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 552 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 553 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 554 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 555 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 556 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 557 558 // Verify we don't accidentally get others 559 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 560 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); 561 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); 562 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 563 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); 564 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); 565 566 // Ensure maxVersions in query is respected 567 Get get = new Get(ROW); 568 get.addColumn(FAMILY, QUALIFIER); 569 get.readVersions(2); 570 Result result = ht.get(get); 571 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] }, 572 new byte[][] { VALUES[4], VALUES[5] }, 0, 1); 573 574 Scan scan = new Scan(ROW); 575 scan.addColumn(FAMILY, QUALIFIER); 576 scan.setMaxVersions(2); 577 result = getSingleScanResult(ht, scan); 578 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] }, 579 new byte[][] { VALUES[4], VALUES[5] }, 0, 1); 580 581 // Flush and redo 582 583 TEST_UTIL.flush(); 584 585 // Verify we can get each one properly 586 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 587 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 588 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 589 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 590 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 591 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 592 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]); 593 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]); 594 595 // Verify we don't accidentally get others 596 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 597 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); 598 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); 599 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 600 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]); 601 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]); 602 603 // Ensure maxVersions in query is respected 604 get = new Get(ROW); 605 get.addColumn(FAMILY, QUALIFIER); 606 get.readVersions(2); 607 result = ht.get(get); 608 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] }, 609 new byte[][] { VALUES[4], VALUES[5] }, 0, 1); 610 611 scan = new Scan(ROW); 612 scan.addColumn(FAMILY, QUALIFIER); 613 scan.setMaxVersions(2); 614 result = getSingleScanResult(ht, scan); 615 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] }, 616 new byte[][] { VALUES[4], VALUES[5] }, 0, 1); 617 618 // Add some memstore and retest 619 620 // Insert 4 more versions of same column and a dupe 621 put = new Put(ROW); 622 put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]); 623 put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]); 624 put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]); 625 put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]); 626 put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]); 627 ht.put(put); 628 629 // Ensure maxVersions in query is respected 630 get = new Get(ROW); 631 get.addColumn(FAMILY, QUALIFIER); 632 get.readVersions(7); 633 result = ht.get(get); 634 assertNResult(result, ROW, FAMILY, QUALIFIER, 635 new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] }, 636 new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], 637 VALUES[8] }, 0, 6); 638 639 scan = new Scan(ROW); 640 scan.addColumn(FAMILY, QUALIFIER); 641 scan.setMaxVersions(7); 642 result = getSingleScanResult(ht, scan); 643 assertNResult(result, ROW, FAMILY, QUALIFIER, 644 new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] }, 645 new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], 646 VALUES[8] }, 0, 6); 647 648 get = new Get(ROW); 649 get.readVersions(7); 650 result = ht.get(get); 651 assertNResult(result, ROW, FAMILY, QUALIFIER, 652 new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] }, 653 new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], 654 VALUES[8] }, 0, 6); 655 656 scan = new Scan(ROW); 657 scan.setMaxVersions(7); 658 result = getSingleScanResult(ht, scan); 659 assertNResult(result, ROW, FAMILY, QUALIFIER, 660 new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] }, 661 new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], 662 VALUES[8] }, 0, 6); 663 664 // Verify we can get each one properly 665 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 666 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 667 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]); 668 getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]); 669 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]); 670 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]); 671 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]); 672 scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]); 673 674 // Verify we don't accidentally get others 675 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 676 getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]); 677 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]); 678 scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]); 679 680 // Ensure maxVersions of table is respected 681 682 TEST_UTIL.flush(); 683 684 // Insert 4 more versions of same column and a dupe 685 put = new Put(ROW); 686 put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]); 687 put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]); 688 put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]); 689 put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]); 690 ht.put(put); 691 692 get = new Get(ROW); 693 get.addColumn(FAMILY, QUALIFIER); 694 get.readVersions(Integer.MAX_VALUE); 695 result = ht.get(get); 696 assertNResult(result, ROW, FAMILY, QUALIFIER, 697 new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], 698 STAMPS[11], STAMPS[13], STAMPS[15] }, 699 new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], 700 VALUES[11], VALUES[13], VALUES[15] }, 0, 9); 701 702 scan = new Scan(ROW); 703 scan.addColumn(FAMILY, QUALIFIER); 704 scan.setMaxVersions(Integer.MAX_VALUE); 705 result = getSingleScanResult(ht, scan); 706 assertNResult(result, ROW, FAMILY, QUALIFIER, 707 new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], 708 STAMPS[11], STAMPS[13], STAMPS[15] }, 709 new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], 710 VALUES[11], VALUES[13], VALUES[15] }, 0, 9); 711 712 // Delete a version in the memstore and a version in a storefile 713 Delete delete = new Delete(ROW); 714 delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]); 715 delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]); 716 ht.delete(delete); 717 718 // Test that it's gone 719 get = new Get(ROW); 720 get.addColumn(FAMILY, QUALIFIER); 721 get.readVersions(Integer.MAX_VALUE); 722 result = ht.get(get); 723 assertNResult(result, ROW, FAMILY, QUALIFIER, 724 new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], 725 STAMPS[9], STAMPS[13], STAMPS[15] }, 726 new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], 727 VALUES[9], VALUES[13], VALUES[15] }, 0, 9); 728 729 scan = new Scan(ROW); 730 scan.addColumn(FAMILY, QUALIFIER); 731 scan.setMaxVersions(Integer.MAX_VALUE); 732 result = getSingleScanResult(ht, scan); 733 assertNResult(result, ROW, FAMILY, QUALIFIER, 734 new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], 735 STAMPS[9], STAMPS[13], STAMPS[15] }, 736 new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], 737 VALUES[9], VALUES[13], VALUES[15] }, 0, 9); 738 } 739 } 740 741 @Test public void testUpdates() throws Exception { 742 final TableName tableName = name.getTableName(); 743 try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10)) { 744 745 // Write a column with values at timestamp 1, 2 and 3 746 byte[] row = Bytes.toBytes("row1"); 747 byte[] qualifier = Bytes.toBytes("myCol"); 748 Put put = new Put(row); 749 put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA")); 750 hTable.put(put); 751 752 put = new Put(row); 753 put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB")); 754 hTable.put(put); 755 756 put = new Put(row); 757 put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE")); 758 hTable.put(put); 759 760 Get get = new Get(row); 761 get.addColumn(FAMILY, qualifier); 762 get.readAllVersions(); 763 764 // Check that the column indeed has the right values at timestamps 1 and 765 // 2 766 Result result = hTable.get(get); 767 NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier); 768 assertEquals("AAA", Bytes.toString(navigableMap.get(1L))); 769 assertEquals("BBB", Bytes.toString(navigableMap.get(2L))); 770 771 // Update the value at timestamp 1 772 put = new Put(row); 773 put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC")); 774 hTable.put(put); 775 776 // Update the value at timestamp 2 777 put = new Put(row); 778 put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD")); 779 hTable.put(put); 780 781 // Check that the values at timestamp 2 and 1 got updated 782 result = hTable.get(get); 783 navigableMap = result.getMap().get(FAMILY).get(qualifier); 784 assertEquals("CCC", Bytes.toString(navigableMap.get(1L))); 785 assertEquals("DDD", Bytes.toString(navigableMap.get(2L))); 786 } 787 } 788 789 @Test public void testUpdatesWithMajorCompaction() throws Exception { 790 final TableName tableName = name.getTableName(); 791 try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10); 792 Admin admin = TEST_UTIL.getAdmin()) { 793 794 // Write a column with values at timestamp 1, 2 and 3 795 byte[] row = Bytes.toBytes("row2"); 796 byte[] qualifier = Bytes.toBytes("myCol"); 797 Put put = new Put(row); 798 put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA")); 799 hTable.put(put); 800 801 put = new Put(row); 802 put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB")); 803 hTable.put(put); 804 805 put = new Put(row); 806 put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE")); 807 hTable.put(put); 808 809 Get get = new Get(row); 810 get.addColumn(FAMILY, qualifier); 811 get.readAllVersions(); 812 813 // Check that the column indeed has the right values at timestamps 1 and 814 // 2 815 Result result = hTable.get(get); 816 NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier); 817 assertEquals("AAA", Bytes.toString(navigableMap.get(1L))); 818 assertEquals("BBB", Bytes.toString(navigableMap.get(2L))); 819 820 // Trigger a major compaction 821 admin.flush(tableName); 822 admin.majorCompact(tableName); 823 Thread.sleep(6000); 824 825 // Update the value at timestamp 1 826 put = new Put(row); 827 put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC")); 828 hTable.put(put); 829 830 // Update the value at timestamp 2 831 put = new Put(row); 832 put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD")); 833 hTable.put(put); 834 835 // Trigger a major compaction 836 admin.flush(tableName); 837 admin.majorCompact(tableName); 838 Thread.sleep(6000); 839 840 // Check that the values at timestamp 2 and 1 got updated 841 result = hTable.get(get); 842 navigableMap = result.getMap().get(FAMILY).get(qualifier); 843 assertEquals("CCC", Bytes.toString(navigableMap.get(1L))); 844 assertEquals("DDD", Bytes.toString(navigableMap.get(2L))); 845 } 846 } 847 848 @Test public void testMajorCompactionBetweenTwoUpdates() throws Exception { 849 final TableName tableName = name.getTableName(); 850 try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10); 851 Admin admin = TEST_UTIL.getAdmin()) { 852 853 // Write a column with values at timestamp 1, 2 and 3 854 byte[] row = Bytes.toBytes("row3"); 855 byte[] qualifier = Bytes.toBytes("myCol"); 856 Put put = new Put(row); 857 put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA")); 858 hTable.put(put); 859 860 put = new Put(row); 861 put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB")); 862 hTable.put(put); 863 864 put = new Put(row); 865 put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE")); 866 hTable.put(put); 867 868 Get get = new Get(row); 869 get.addColumn(FAMILY, qualifier); 870 get.readAllVersions(); 871 872 // Check that the column indeed has the right values at timestamps 1 and 873 // 2 874 Result result = hTable.get(get); 875 NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier); 876 assertEquals("AAA", Bytes.toString(navigableMap.get(1L))); 877 assertEquals("BBB", Bytes.toString(navigableMap.get(2L))); 878 879 // Trigger a major compaction 880 admin.flush(tableName); 881 admin.majorCompact(tableName); 882 Thread.sleep(6000); 883 884 // Update the value at timestamp 1 885 put = new Put(row); 886 put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC")); 887 hTable.put(put); 888 889 // Trigger a major compaction 890 admin.flush(tableName); 891 admin.majorCompact(tableName); 892 Thread.sleep(6000); 893 894 // Update the value at timestamp 2 895 put = new Put(row); 896 put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD")); 897 hTable.put(put); 898 899 // Trigger a major compaction 900 admin.flush(tableName); 901 admin.majorCompact(tableName); 902 Thread.sleep(6000); 903 904 // Check that the values at timestamp 2 and 1 got updated 905 result = hTable.get(get); 906 navigableMap = result.getMap().get(FAMILY).get(qualifier); 907 908 assertEquals("CCC", Bytes.toString(navigableMap.get(1L))); 909 assertEquals("DDD", Bytes.toString(navigableMap.get(2L))); 910 } 911 } 912 913 @Test public void testGet_EmptyTable() throws IOException { 914 try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) { 915 Get get = new Get(ROW); 916 get.addFamily(FAMILY); 917 Result r = table.get(get); 918 assertTrue(r.isEmpty()); 919 } 920 } 921 922 @Test public void testGet_NullQualifier() throws IOException { 923 try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) { 924 Put put = new Put(ROW); 925 put.addColumn(FAMILY, QUALIFIER, VALUE); 926 table.put(put); 927 928 put = new Put(ROW); 929 put.addColumn(FAMILY, null, VALUE); 930 table.put(put); 931 LOG.info("Row put"); 932 933 Get get = new Get(ROW); 934 get.addColumn(FAMILY, null); 935 Result r = table.get(get); 936 assertEquals(1, r.size()); 937 938 get = new Get(ROW); 939 get.addFamily(FAMILY); 940 r = table.get(get); 941 assertEquals(2, r.size()); 942 } 943 } 944 945 @Test public void testGet_NonExistentRow() throws IOException { 946 try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) { 947 Put put = new Put(ROW); 948 put.addColumn(FAMILY, QUALIFIER, VALUE); 949 table.put(put); 950 LOG.info("Row put"); 951 952 Get get = new Get(ROW); 953 get.addFamily(FAMILY); 954 Result r = table.get(get); 955 assertFalse(r.isEmpty()); 956 System.out.println("Row retrieved successfully"); 957 958 byte[] missingrow = Bytes.toBytes("missingrow"); 959 get = new Get(missingrow); 960 get.addFamily(FAMILY); 961 r = table.get(get); 962 assertTrue(r.isEmpty()); 963 LOG.info("Row missing as it should be"); 964 } 965 } 966 967 @Test public void testPut() throws IOException { 968 final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents"); 969 final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam"); 970 final byte[] row1 = Bytes.toBytes("row1"); 971 final byte[] row2 = Bytes.toBytes("row2"); 972 final byte[] value = Bytes.toBytes("abcd"); 973 try (Table table = TEST_UTIL 974 .createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) { 975 Put put = new Put(row1); 976 put.addColumn(CONTENTS_FAMILY, null, value); 977 table.put(put); 978 979 put = new Put(row2); 980 put.addColumn(CONTENTS_FAMILY, null, value); 981 982 assertEquals(1, put.size()); 983 assertEquals(1, put.getFamilyCellMap().get(CONTENTS_FAMILY).size()); 984 985 // KeyValue v1 expectation. Cast for now until we go all Cell all the time. TODO 986 KeyValue kv = (KeyValue) put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0); 987 988 assertTrue(Bytes.equals(CellUtil.cloneFamily(kv), CONTENTS_FAMILY)); 989 // will it return null or an empty byte array? 990 assertTrue(Bytes.equals(CellUtil.cloneQualifier(kv), new byte[0])); 991 992 assertTrue(Bytes.equals(CellUtil.cloneValue(kv), value)); 993 994 table.put(put); 995 996 Scan scan = new Scan(); 997 scan.addColumn(CONTENTS_FAMILY, null); 998 try (ResultScanner scanner = table.getScanner(scan)) { 999 for (Result r : scanner) { 1000 for (Cell key : r.rawCells()) { 1001 System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString()); 1002 } 1003 } 1004 } 1005 } 1006 } 1007 1008 @Test public void testPutNoCF() throws IOException { 1009 final byte[] BAD_FAM = Bytes.toBytes("BAD_CF"); 1010 final byte[] VAL = Bytes.toBytes(100); 1011 try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) { 1012 boolean caughtNSCFE = false; 1013 1014 try { 1015 Put p = new Put(ROW); 1016 p.addColumn(BAD_FAM, QUALIFIER, VAL); 1017 table.put(p); 1018 } catch (Exception e) { 1019 caughtNSCFE = e instanceof NoSuchColumnFamilyException; 1020 } 1021 assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE); 1022 } 1023 } 1024 1025 @Test public void testRowsPut() throws IOException { 1026 final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents"); 1027 final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam"); 1028 final int NB_BATCH_ROWS = 10; 1029 final byte[] value = Bytes.toBytes("abcd"); 1030 try (Table table = TEST_UTIL 1031 .createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) { 1032 ArrayList<Put> rowsUpdate = new ArrayList<>(); 1033 for (int i = 0; i < NB_BATCH_ROWS; i++) { 1034 byte[] row = Bytes.toBytes("row" + i); 1035 Put put = new Put(row); 1036 put.setDurability(Durability.SKIP_WAL); 1037 put.addColumn(CONTENTS_FAMILY, null, value); 1038 rowsUpdate.add(put); 1039 } 1040 table.put(rowsUpdate); 1041 Scan scan = new Scan(); 1042 scan.addFamily(CONTENTS_FAMILY); 1043 try (ResultScanner scanner = table.getScanner(scan)) { 1044 int nbRows = 0; 1045 for (@SuppressWarnings("unused") Result row : scanner) { 1046 nbRows++; 1047 } 1048 assertEquals(NB_BATCH_ROWS, nbRows); 1049 } 1050 } 1051 } 1052 1053 @Test public void testRowsPutBufferedManyManyFlushes() throws IOException { 1054 final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents"); 1055 final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam"); 1056 final byte[] value = Bytes.toBytes("abcd"); 1057 final int NB_BATCH_ROWS = 10; 1058 try (Table table = TEST_UTIL 1059 .createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) { 1060 ArrayList<Put> rowsUpdate = new ArrayList<>(); 1061 for (int i = 0; i < NB_BATCH_ROWS * 10; i++) { 1062 byte[] row = Bytes.toBytes("row" + i); 1063 Put put = new Put(row); 1064 put.setDurability(Durability.SKIP_WAL); 1065 put.addColumn(CONTENTS_FAMILY, null, value); 1066 rowsUpdate.add(put); 1067 } 1068 table.put(rowsUpdate); 1069 1070 Scan scan = new Scan(); 1071 scan.addFamily(CONTENTS_FAMILY); 1072 try (ResultScanner scanner = table.getScanner(scan)) { 1073 int nbRows = 0; 1074 for (@SuppressWarnings("unused") Result row : scanner) { 1075 nbRows++; 1076 } 1077 assertEquals(NB_BATCH_ROWS * 10, nbRows); 1078 } 1079 } 1080 } 1081 1082 @Test public void testAddKeyValue() { 1083 final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents"); 1084 final byte[] value = Bytes.toBytes("abcd"); 1085 final byte[] row1 = Bytes.toBytes("row1"); 1086 final byte[] row2 = Bytes.toBytes("row2"); 1087 byte[] qualifier = Bytes.toBytes("qf1"); 1088 Put put = new Put(row1); 1089 1090 // Adding KeyValue with the same row 1091 KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value); 1092 boolean ok = true; 1093 try { 1094 put.add(kv); 1095 } catch (IOException e) { 1096 ok = false; 1097 } 1098 assertTrue(ok); 1099 1100 // Adding KeyValue with the different row 1101 kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value); 1102 ok = false; 1103 try { 1104 put.add(kv); 1105 } catch (IOException e) { 1106 ok = true; 1107 } 1108 assertTrue(ok); 1109 } 1110 1111 /** 1112 * test for HBASE-737 1113 */ 1114 @Test public void testHBase737() throws IOException { 1115 final byte[] FAM1 = Bytes.toBytes("fam1"); 1116 final byte[] FAM2 = Bytes.toBytes("fam2"); 1117 // Open table 1118 try (Table table = TEST_UTIL.createTable(name.getTableName(), new byte[][] { FAM1, FAM2 })) { 1119 // Insert some values 1120 Put put = new Put(ROW); 1121 put.addColumn(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg")); 1122 table.put(put); 1123 try { 1124 Thread.sleep(1000); 1125 } catch (InterruptedException i) { 1126 //ignore 1127 } 1128 1129 put = new Put(ROW); 1130 put.addColumn(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456")); 1131 table.put(put); 1132 1133 try { 1134 Thread.sleep(1000); 1135 } catch (InterruptedException i) { 1136 //ignore 1137 } 1138 1139 put = new Put(ROW); 1140 put.addColumn(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop")); 1141 table.put(put); 1142 1143 long[] times = new long[3]; 1144 1145 // First scan the memstore 1146 1147 Scan scan = new Scan(); 1148 scan.addFamily(FAM1); 1149 scan.addFamily(FAM2); 1150 try (ResultScanner s = table.getScanner(scan)) { 1151 int index = 0; 1152 Result r; 1153 while ((r = s.next()) != null) { 1154 for (Cell key : r.rawCells()) { 1155 times[index++] = key.getTimestamp(); 1156 } 1157 } 1158 } 1159 for (int i = 0; i < times.length - 1; i++) { 1160 for (int j = i + 1; j < times.length; j++) { 1161 assertTrue(times[j] > times[i]); 1162 } 1163 } 1164 1165 // Flush data to disk and try again 1166 TEST_UTIL.flush(); 1167 1168 // Reset times 1169 Arrays.fill(times, 0); 1170 1171 try { 1172 Thread.sleep(1000); 1173 } catch (InterruptedException i) { 1174 //ignore 1175 } 1176 scan = new Scan(); 1177 scan.addFamily(FAM1); 1178 scan.addFamily(FAM2); 1179 try (ResultScanner s = table.getScanner(scan)) { 1180 int index = 0; 1181 Result r = null; 1182 while ((r = s.next()) != null) { 1183 for (Cell key : r.rawCells()) { 1184 times[index++] = key.getTimestamp(); 1185 } 1186 } 1187 for (int i = 0; i < times.length - 1; i++) { 1188 for (int j = i + 1; j < times.length; j++) { 1189 assertTrue(times[j] > times[i]); 1190 } 1191 } 1192 } 1193 } 1194 } 1195 1196 @Test public void testListTables() throws IOException { 1197 final String testTableName = name.getTableName().toString(); 1198 final TableName tableName1 = TableName.valueOf(testTableName + "1"); 1199 final TableName tableName2 = TableName.valueOf(testTableName + "2"); 1200 final TableName tableName3 = TableName.valueOf(testTableName + "3"); 1201 TableName[] tables = new TableName[] { tableName1, tableName2, tableName3 }; 1202 for (TableName table : tables) { 1203 TEST_UTIL.createTable(table, FAMILY); 1204 } 1205 try (Admin admin = TEST_UTIL.getAdmin()) { 1206 List<TableDescriptor> ts = admin.listTableDescriptors(); 1207 HashSet<TableDescriptor> result = new HashSet<>(ts); 1208 int size = result.size(); 1209 assertTrue(size >= tables.length); 1210 for (TableName table : tables) { 1211 boolean found = false; 1212 for (TableDescriptor t : ts) { 1213 if (t.getTableName().equals(table)) { 1214 found = true; 1215 break; 1216 } 1217 } 1218 assertTrue("Not found: " + table, found); 1219 } 1220 } 1221 } 1222 1223 /** 1224 * simple test that just executes parts of the client 1225 * API that accept a pre-created Connection instance 1226 */ 1227 @Test public void testUnmanagedHConnection() throws IOException { 1228 final TableName tableName = name.getTableName(); 1229 TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY); 1230 try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); 1231 Table t = conn.getTable(tableName); 1232 Admin admin = conn.getAdmin()) { 1233 assertTrue(admin.tableExists(tableName)); 1234 assertTrue(t.get(new Get(ROW)).isEmpty()); 1235 } 1236 } 1237 1238 /** 1239 * test of that unmanaged HConnections are able to reconnect 1240 * properly (see HBASE-5058) 1241 */ 1242 @Test public void testUnmanagedHConnectionReconnect() throws Exception { 1243 Configuration conf = TEST_UTIL.getConfiguration(); 1244 Class registryImpl = conf 1245 .getClass(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, ZKConnectionRegistry.class); 1246 // This test does not make sense for MasterRegistry since it stops the only master in the 1247 // cluster and starts a new master without populating the underlying config for the connection. 1248 Assume.assumeFalse(registryImpl.equals(MasterRegistry.class)); 1249 final TableName tableName = name.getTableName(); 1250 TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY); 1251 try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) { 1252 try (Table t = conn.getTable(tableName); Admin admin = conn.getAdmin()) { 1253 assertTrue(admin.tableExists(tableName)); 1254 assertTrue(t.get(new Get(ROW)).isEmpty()); 1255 } 1256 1257 // stop the master 1258 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); 1259 cluster.stopMaster(0, false); 1260 cluster.waitOnMaster(0); 1261 1262 // start up a new master 1263 cluster.startMaster(); 1264 assertTrue(cluster.waitForActiveAndReadyMaster()); 1265 1266 // test that the same unmanaged connection works with a new 1267 // Admin and can connect to the new master; 1268 boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration()); 1269 try (Admin admin = conn.getAdmin()) { 1270 assertTrue(admin.tableExists(tableName)); 1271 assertEquals( 1272 admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size(), 1273 SLAVES + (tablesOnMaster ? 1 : 0)); 1274 } 1275 } 1276 } 1277 1278 @Test public void testMiscHTableStuff() throws IOException { 1279 final String testTableName = name.getTableName().toString(); 1280 final TableName tableAname = TableName.valueOf(testTableName + "A"); 1281 final TableName tableBname = TableName.valueOf(testTableName + "B"); 1282 final byte[] attrName = Bytes.toBytes("TESTATTR"); 1283 final byte[] attrValue = Bytes.toBytes("somevalue"); 1284 byte[] value = Bytes.toBytes("value"); 1285 1286 try (Table a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY); 1287 Table b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY)) { 1288 Put put = new Put(ROW); 1289 put.addColumn(HConstants.CATALOG_FAMILY, null, value); 1290 a.put(put); 1291 1292 // open a new connection to A and a connection to b 1293 try (Table newA = TEST_UTIL.getConnection().getTable(tableAname)) { 1294 1295 // copy data from A to B 1296 Scan scan = new Scan(); 1297 scan.addFamily(HConstants.CATALOG_FAMILY); 1298 try (ResultScanner s = newA.getScanner(scan)) { 1299 for (Result r : s) { 1300 put = new Put(r.getRow()); 1301 put.setDurability(Durability.SKIP_WAL); 1302 for (Cell kv : r.rawCells()) { 1303 put.add(kv); 1304 } 1305 b.put(put); 1306 } 1307 } 1308 } 1309 1310 // Opening a new connection to A will cause the tables to be reloaded 1311 try (Table anotherA = TEST_UTIL.getConnection().getTable(tableAname)) { 1312 Get get = new Get(ROW); 1313 get.addFamily(HConstants.CATALOG_FAMILY); 1314 anotherA.get(get); 1315 } 1316 1317 // We can still access A through newA because it has the table information 1318 // cached. And if it needs to recalibrate, that will cause the information 1319 // to be reloaded. 1320 1321 // Test user metadata 1322 Admin admin = TEST_UTIL.getAdmin(); 1323 // make a modifiable descriptor 1324 HTableDescriptor desc = new HTableDescriptor(a.getDescriptor()); 1325 // offline the table 1326 admin.disableTable(tableAname); 1327 // add a user attribute to HTD 1328 desc.setValue(attrName, attrValue); 1329 // add a user attribute to HCD 1330 for (HColumnDescriptor c : desc.getFamilies()) { 1331 c.setValue(attrName, attrValue); 1332 } 1333 // update metadata for all regions of this table 1334 admin.modifyTable(desc); 1335 // enable the table 1336 admin.enableTable(tableAname); 1337 1338 // Test that attribute changes were applied 1339 desc = new HTableDescriptor(a.getDescriptor()); 1340 assertEquals("wrong table descriptor returned", desc.getTableName(), tableAname); 1341 // check HTD attribute 1342 value = desc.getValue(attrName); 1343 assertNotNull("missing HTD attribute value", value); 1344 assertFalse("HTD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0); 1345 // check HCD attribute 1346 for (HColumnDescriptor c : desc.getFamilies()) { 1347 value = c.getValue(attrName); 1348 assertNotNull("missing HCD attribute value", value); 1349 assertFalse("HCD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0); 1350 } 1351 } 1352 } 1353} 1354