001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.client; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNotEquals; 022import static org.junit.Assert.assertNotNull; 023import static org.junit.Assert.assertTrue; 024import static org.junit.Assert.fail; 025 026import java.io.IOException; 027import java.util.ArrayList; 028import java.util.HashMap; 029import java.util.List; 030import java.util.Map; 031import org.apache.hadoop.conf.Configuration; 032import org.apache.hadoop.hbase.Cell; 033import org.apache.hadoop.hbase.CellUtil; 034import org.apache.hadoop.hbase.DoNotRetryIOException; 035import org.apache.hadoop.hbase.HBaseClassTestRule; 036import org.apache.hadoop.hbase.HBaseTestingUtility; 037import org.apache.hadoop.hbase.HConstants; 038import org.apache.hadoop.hbase.HTableDescriptor; 039import org.apache.hadoop.hbase.KeyValue; 040import org.apache.hadoop.hbase.TableName; 041import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; 042import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint; 043import org.apache.hadoop.hbase.testclassification.LargeTests; 044import org.apache.hadoop.hbase.util.Bytes; 045import org.junit.AfterClass; 046import org.junit.BeforeClass; 047import org.junit.ClassRule; 048import org.junit.Rule; 049import org.junit.Test; 050import org.junit.experimental.categories.Category; 051import org.junit.rules.TestName; 052import org.slf4j.Logger; 053import org.slf4j.LoggerFactory; 054 055/** 056 * Run Increment tests that use the HBase clients; {@link HTable}. 057 * 058 * Test is parameterized to run the slow and fast increment code paths. If fast, in the @before, we 059 * do a rolling restart of the single regionserver so that it can pick up the go fast configuration. 060 * Doing it this way should be faster than starting/stopping a cluster per test. 061 * 062 * Test takes a long time because spin up a cluster between each run -- ugh. 063 */ 064@Category(LargeTests.class) 065public class TestIncrementsFromClientSide { 066 067 @ClassRule 068 public static final HBaseClassTestRule CLASS_RULE = 069 HBaseClassTestRule.forClass(TestIncrementsFromClientSide.class); 070 071 final Logger LOG = LoggerFactory.getLogger(getClass()); 072 protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 073 private static byte [] ROW = Bytes.toBytes("testRow"); 074 private static byte [] FAMILY = Bytes.toBytes("testFamily"); 075 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); 076 // This test depends on there being only one slave running at at a time. See the @Before 077 // method where we do rolling restart. 078 protected static int SLAVES = 1; 079 @Rule public TestName name = new TestName(); 080 081 @BeforeClass 082 public static void beforeClass() throws Exception { 083 Configuration conf = TEST_UTIL.getConfiguration(); 084 conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, 085 MultiRowMutationEndpoint.class.getName()); 086 conf.setBoolean("hbase.table.sanity.checks", true); // enable for below tests 087 // We need more than one region server in this test 088 TEST_UTIL.startMiniCluster(SLAVES); 089 } 090 091 /** 092 * @throws java.lang.Exception 093 */ 094 @AfterClass 095 public static void afterClass() throws Exception { 096 TEST_UTIL.shutdownMiniCluster(); 097 } 098 099 /** 100 * Test increment result when there are duplicate rpc request. 101 */ 102 @Test 103 public void testDuplicateIncrement() throws Exception { 104 HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName())); 105 Map<String, String> kvs = new HashMap<>(); 106 kvs.put(HConnectionTestingUtility.SleepAtFirstRpcCall.SLEEP_TIME_CONF_KEY, "2000"); 107 hdt.addCoprocessor(HConnectionTestingUtility.SleepAtFirstRpcCall.class.getName(), null, 1, kvs); 108 TEST_UTIL.createTable(hdt, new byte[][] { ROW }).close(); 109 110 Configuration c = new Configuration(TEST_UTIL.getConfiguration()); 111 c.setInt(HConstants.HBASE_CLIENT_PAUSE, 50); 112 // Client will retry beacuse rpc timeout is small than the sleep time of first rpc call 113 c.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500); 114 115 Connection connection = ConnectionFactory.createConnection(c); 116 Table t = connection.getTable(TableName.valueOf(name.getMethodName())); 117 if (t instanceof HTable) { 118 HTable table = (HTable) t; 119 table.setOperationTimeout(3 * 1000); 120 121 try { 122 Increment inc = new Increment(ROW); 123 inc.addColumn(TEST_UTIL.fam1, QUALIFIER, 1); 124 Result result = table.increment(inc); 125 126 Cell [] cells = result.rawCells(); 127 assertEquals(1, cells.length); 128 assertIncrementKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, 1); 129 130 // Verify expected result 131 Result readResult = table.get(new Get(ROW)); 132 cells = readResult.rawCells(); 133 assertEquals(1, cells.length); 134 assertIncrementKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, 1); 135 } finally { 136 table.close(); 137 connection.close(); 138 } 139 } 140 } 141 142 @Test 143 public void testIncrementWithDeletes() throws Exception { 144 LOG.info("Starting " + this.name.getMethodName()); 145 final TableName TABLENAME = 146 TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName())); 147 Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY); 148 final byte[] COLUMN = Bytes.toBytes("column"); 149 150 ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5); 151 TEST_UTIL.flush(TABLENAME); 152 153 Delete del = new Delete(ROW); 154 ht.delete(del); 155 156 ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5); 157 158 Get get = new Get(ROW); 159 Result r = ht.get(get); 160 assertEquals(1, r.size()); 161 assertEquals(5, Bytes.toLong(r.getValue(FAMILY, COLUMN))); 162 } 163 164 @Test 165 public void testIncrementingInvalidValue() throws Exception { 166 LOG.info("Starting " + this.name.getMethodName()); 167 final TableName TABLENAME = 168 TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName())); 169 Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY); 170 final byte[] COLUMN = Bytes.toBytes("column"); 171 Put p = new Put(ROW); 172 // write an integer here (not a Long) 173 p.addColumn(FAMILY, COLUMN, Bytes.toBytes(5)); 174 ht.put(p); 175 try { 176 ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5); 177 fail("Should have thrown DoNotRetryIOException"); 178 } catch (DoNotRetryIOException iox) { 179 // success 180 } 181 Increment inc = new Increment(ROW); 182 inc.addColumn(FAMILY, COLUMN, 5); 183 try { 184 ht.increment(inc); 185 fail("Should have thrown DoNotRetryIOException"); 186 } catch (DoNotRetryIOException iox) { 187 // success 188 } 189 } 190 191 @Test 192 public void testBatchIncrementsWithReturnResultFalse() throws Exception { 193 LOG.info("Starting testBatchIncrementsWithReturnResultFalse"); 194 final TableName tableName = TableName.valueOf(name.getMethodName()); 195 Table table = TEST_UTIL.createTable(tableName, FAMILY); 196 Increment inc1 = new Increment(Bytes.toBytes("row2")); 197 inc1.setReturnResults(false); 198 inc1.addColumn(FAMILY, Bytes.toBytes("f1"), 1); 199 Increment inc2 = new Increment(Bytes.toBytes("row2")); 200 inc2.setReturnResults(false); 201 inc2.addColumn(FAMILY, Bytes.toBytes("f1"), 1); 202 List<Increment> incs = new ArrayList<>(); 203 incs.add(inc1); 204 incs.add(inc2); 205 Object[] results = new Object[2]; 206 table.batch(incs, results); 207 assertTrue(results.length == 2); 208 for(Object r : results) { 209 Result result = (Result)r; 210 assertTrue(result.isEmpty()); 211 } 212 table.close(); 213 } 214 215 @Test 216 public void testIncrementInvalidArguments() throws Exception { 217 LOG.info("Starting " + this.name.getMethodName()); 218 final TableName TABLENAME = 219 TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName())); 220 Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY); 221 final byte[] COLUMN = Bytes.toBytes("column"); 222 try { 223 // try null row 224 ht.incrementColumnValue(null, FAMILY, COLUMN, 5); 225 fail("Should have thrown IOException"); 226 } catch (IOException iox) { 227 // success 228 } 229 try { 230 // try null family 231 ht.incrementColumnValue(ROW, null, COLUMN, 5); 232 fail("Should have thrown IOException"); 233 } catch (IOException iox) { 234 // success 235 } 236 // try null row 237 try { 238 Increment incNoRow = new Increment((byte [])null); 239 incNoRow.addColumn(FAMILY, COLUMN, 5); 240 fail("Should have thrown IllegalArgumentException"); 241 } catch (IllegalArgumentException iax) { 242 // success 243 } catch (NullPointerException npe) { 244 // success 245 } 246 // try null family 247 try { 248 Increment incNoFamily = new Increment(ROW); 249 incNoFamily.addColumn(null, COLUMN, 5); 250 fail("Should have thrown IllegalArgumentException"); 251 } catch (IllegalArgumentException iax) { 252 // success 253 } 254 } 255 256 @Test 257 public void testIncrementOutOfOrder() throws Exception { 258 LOG.info("Starting " + this.name.getMethodName()); 259 final TableName TABLENAME = 260 TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName())); 261 Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY); 262 263 byte [][] QUALIFIERS = new byte [][] { 264 Bytes.toBytes("B"), Bytes.toBytes("A"), Bytes.toBytes("C") 265 }; 266 267 Increment inc = new Increment(ROW); 268 for (int i=0; i<QUALIFIERS.length; i++) { 269 inc.addColumn(FAMILY, QUALIFIERS[i], 1); 270 } 271 ht.increment(inc); 272 273 // Verify expected results 274 Get get = new Get(ROW); 275 Result r = ht.get(get); 276 Cell [] kvs = r.rawCells(); 277 assertEquals(3, kvs.length); 278 assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 1); 279 assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 1); 280 assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1); 281 282 // Now try multiple columns again 283 inc = new Increment(ROW); 284 for (int i=0; i<QUALIFIERS.length; i++) { 285 inc.addColumn(FAMILY, QUALIFIERS[i], 1); 286 } 287 ht.increment(inc); 288 289 // Verify 290 r = ht.get(get); 291 kvs = r.rawCells(); 292 assertEquals(3, kvs.length); 293 assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 2); 294 assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 2); 295 assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2); 296 } 297 298 @Test 299 public void testIncrementOnSameColumn() throws Exception { 300 LOG.info("Starting " + this.name.getMethodName()); 301 final byte[] TABLENAME = Bytes.toBytes(filterStringSoTableNameSafe(this.name.getMethodName())); 302 Table ht = TEST_UTIL.createTable(TableName.valueOf(TABLENAME), FAMILY); 303 304 byte[][] QUALIFIERS = 305 new byte[][] { Bytes.toBytes("A"), Bytes.toBytes("B"), Bytes.toBytes("C") }; 306 307 Increment inc = new Increment(ROW); 308 for (int i = 0; i < QUALIFIERS.length; i++) { 309 inc.addColumn(FAMILY, QUALIFIERS[i], 1); 310 inc.addColumn(FAMILY, QUALIFIERS[i], 1); 311 } 312 ht.increment(inc); 313 314 // Verify expected results 315 Get get = new Get(ROW); 316 Result r = ht.get(get); 317 Cell[] kvs = r.rawCells(); 318 assertEquals(3, kvs.length); 319 assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1); 320 assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 1); 321 assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1); 322 323 // Now try multiple columns again 324 inc = new Increment(ROW); 325 for (int i = 0; i < QUALIFIERS.length; i++) { 326 inc.addColumn(FAMILY, QUALIFIERS[i], 1); 327 inc.addColumn(FAMILY, QUALIFIERS[i], 1); 328 } 329 ht.increment(inc); 330 331 // Verify 332 r = ht.get(get); 333 kvs = r.rawCells(); 334 assertEquals(3, kvs.length); 335 assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 2); 336 assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 2); 337 assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2); 338 339 ht.close(); 340 } 341 342 @Test 343 public void testIncrementIncrZeroAtFirst() throws Exception { 344 LOG.info("Starting " + this.name.getMethodName()); 345 final TableName TABLENAME = 346 TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName())); 347 Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY); 348 349 byte[] col1 = Bytes.toBytes("col1"); 350 byte[] col2 = Bytes.toBytes("col2"); 351 byte[] col3 = Bytes.toBytes("col3"); 352 353 // Now increment zero at first time incr 354 Increment inc = new Increment(ROW); 355 inc.addColumn(FAMILY, col1, 0); 356 ht.increment(inc); 357 358 // Verify expected results 359 Get get = new Get(ROW); 360 Result r = ht.get(get); 361 Cell [] kvs = r.rawCells(); 362 assertEquals(1, kvs.length); 363 assertNotNull(kvs[0]); 364 assertIncrementKey(kvs[0], ROW, FAMILY, col1, 0); 365 366 // Now try multiple columns by different amounts 367 inc = new Increment(ROW); 368 inc.addColumn(FAMILY, col1, 1); 369 inc.addColumn(FAMILY, col2, 0); 370 inc.addColumn(FAMILY, col3, 2); 371 ht.increment(inc); 372 // Verify 373 get = new Get(ROW); 374 r = ht.get(get); 375 kvs = r.rawCells(); 376 assertEquals(3, kvs.length); 377 assertNotNull(kvs[0]); 378 assertNotNull(kvs[1]); 379 assertNotNull(kvs[2]); 380 assertIncrementKey(kvs[0], ROW, FAMILY, col1, 1); 381 assertIncrementKey(kvs[1], ROW, FAMILY, col2, 0); 382 assertIncrementKey(kvs[2], ROW, FAMILY, col3, 2); 383 } 384 385 @Test 386 public void testIncrement() throws Exception { 387 LOG.info("Starting " + this.name.getMethodName()); 388 final TableName TABLENAME = 389 TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName())); 390 Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY); 391 392 byte [][] ROWS = new byte [][] { 393 Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"), 394 Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"), 395 Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i") 396 }; 397 byte [][] QUALIFIERS = new byte [][] { 398 Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"), 399 Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"), 400 Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i") 401 }; 402 403 // Do some simple single-column increments 404 405 // First with old API 406 ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[0], 1); 407 ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[1], 2); 408 ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[2], 3); 409 ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[3], 4); 410 411 // Now increment things incremented with old and do some new 412 Increment inc = new Increment(ROW); 413 inc.addColumn(FAMILY, QUALIFIERS[1], 1); 414 inc.addColumn(FAMILY, QUALIFIERS[3], 1); 415 inc.addColumn(FAMILY, QUALIFIERS[4], 1); 416 ht.increment(inc); 417 418 // Verify expected results 419 Get get = new Get(ROW); 420 Result r = ht.get(get); 421 Cell [] kvs = r.rawCells(); 422 assertEquals(5, kvs.length); 423 assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1); 424 assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 3); 425 assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 3); 426 assertIncrementKey(kvs[3], ROW, FAMILY, QUALIFIERS[3], 5); 427 assertIncrementKey(kvs[4], ROW, FAMILY, QUALIFIERS[4], 1); 428 429 // Now try multiple columns by different amounts 430 inc = new Increment(ROWS[0]); 431 for (int i=0;i<QUALIFIERS.length;i++) { 432 inc.addColumn(FAMILY, QUALIFIERS[i], i+1); 433 } 434 ht.increment(inc); 435 // Verify 436 get = new Get(ROWS[0]); 437 r = ht.get(get); 438 kvs = r.rawCells(); 439 assertEquals(QUALIFIERS.length, kvs.length); 440 for (int i=0;i<QUALIFIERS.length;i++) { 441 assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], i+1); 442 } 443 444 // Re-increment them 445 inc = new Increment(ROWS[0]); 446 for (int i=0;i<QUALIFIERS.length;i++) { 447 inc.addColumn(FAMILY, QUALIFIERS[i], i+1); 448 } 449 ht.increment(inc); 450 // Verify 451 r = ht.get(get); 452 kvs = r.rawCells(); 453 assertEquals(QUALIFIERS.length, kvs.length); 454 for (int i=0;i<QUALIFIERS.length;i++) { 455 assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1)); 456 } 457 458 // Verify that an Increment of an amount of zero, returns current count; i.e. same as for above 459 // test, that is: 2 * (i + 1). 460 inc = new Increment(ROWS[0]); 461 for (int i = 0; i < QUALIFIERS.length; i++) { 462 inc.addColumn(FAMILY, QUALIFIERS[i], 0); 463 } 464 ht.increment(inc); 465 r = ht.get(get); 466 kvs = r.rawCells(); 467 assertEquals(QUALIFIERS.length, kvs.length); 468 for (int i = 0; i < QUALIFIERS.length; i++) { 469 assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1)); 470 } 471 } 472 473 @Test 474 public void testIncrementWithCustomTimestamp() throws IOException { 475 TableName TABLENAME = TableName.valueOf(name.getMethodName()); 476 Table table = TEST_UTIL.createTable(TABLENAME, FAMILY); 477 long timestamp = 999; 478 Increment increment = new Increment(ROW); 479 increment.add(CellUtil.createCell(ROW, FAMILY, QUALIFIER, timestamp, KeyValue.Type.Put.getCode(), Bytes.toBytes(100L))); 480 Result r = table.increment(increment); 481 assertEquals(1, r.size()); 482 assertEquals(timestamp, r.rawCells()[0].getTimestamp()); 483 r = table.get(new Get(ROW)); 484 assertEquals(1, r.size()); 485 assertEquals(timestamp, r.rawCells()[0].getTimestamp()); 486 r = table.increment(increment); 487 assertEquals(1, r.size()); 488 assertNotEquals(timestamp, r.rawCells()[0].getTimestamp()); 489 r = table.get(new Get(ROW)); 490 assertEquals(1, r.size()); 491 assertNotEquals(timestamp, r.rawCells()[0].getTimestamp()); 492 } 493 494 /** 495 * Call over to the adjacent class's method of same name. 496 */ 497 static void assertIncrementKey(Cell key, byte [] row, byte [] family, 498 byte [] qualifier, long value) throws Exception { 499 TestFromClientSide.assertIncrementKey(key, row, family, qualifier, value); 500 } 501 502 public static String filterStringSoTableNameSafe(final String str) { 503 return str.replaceAll("\\[fast\\=(.*)\\]", ".FAST.is.$1"); 504 } 505}