001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.client; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNotEquals; 022import static org.junit.Assert.assertNotNull; 023import static org.junit.Assert.assertTrue; 024import static org.junit.Assert.fail; 025 026import java.io.IOException; 027import java.util.ArrayList; 028import java.util.HashMap; 029import java.util.List; 030import java.util.Map; 031import org.apache.hadoop.conf.Configuration; 032import org.apache.hadoop.hbase.Cell; 033import org.apache.hadoop.hbase.CellUtil; 034import org.apache.hadoop.hbase.DoNotRetryIOException; 035import org.apache.hadoop.hbase.HBaseClassTestRule; 036import org.apache.hadoop.hbase.HBaseTestingUtility; 037import org.apache.hadoop.hbase.HConstants; 038import org.apache.hadoop.hbase.HTableDescriptor; 039import org.apache.hadoop.hbase.KeyValue; 040import org.apache.hadoop.hbase.TableName; 041import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; 042import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint; 043import org.apache.hadoop.hbase.testclassification.LargeTests; 044import org.apache.hadoop.hbase.util.Bytes; 045import org.junit.AfterClass; 046import org.junit.BeforeClass; 047import org.junit.ClassRule; 048import org.junit.Rule; 049import org.junit.Test; 050import org.junit.experimental.categories.Category; 051import org.junit.rules.TestName; 052import org.slf4j.Logger; 053import org.slf4j.LoggerFactory; 054 055/** 056 * Run Increment tests that use the HBase clients; {@link HTable}. 057 * 058 * Test is parameterized to run the slow and fast increment code paths. If fast, in the @before, we 059 * do a rolling restart of the single regionserver so that it can pick up the go fast configuration. 060 * Doing it this way should be faster than starting/stopping a cluster per test. 061 * 062 * Test takes a long time because spin up a cluster between each run -- ugh. 063 */ 064@Category(LargeTests.class) 065public class TestIncrementsFromClientSide { 066 067 @ClassRule 068 public static final HBaseClassTestRule CLASS_RULE = 069 HBaseClassTestRule.forClass(TestIncrementsFromClientSide.class); 070 071 final Logger LOG = LoggerFactory.getLogger(getClass()); 072 protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 073 private static byte [] ROW = Bytes.toBytes("testRow"); 074 private static byte [] FAMILY = Bytes.toBytes("testFamily"); 075 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); 076 // This test depends on there being only one slave running at at a time. See the @Before 077 // method where we do rolling restart. 078 protected static int SLAVES = 1; 079 @Rule public TestName name = new TestName(); 080 081 @BeforeClass 082 public static void beforeClass() throws Exception { 083 Configuration conf = TEST_UTIL.getConfiguration(); 084 conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, 085 MultiRowMutationEndpoint.class.getName()); 086 // We need more than one region server in this test 087 TEST_UTIL.startMiniCluster(SLAVES); 088 } 089 090 /** 091 * @throws java.lang.Exception 092 */ 093 @AfterClass 094 public static void afterClass() throws Exception { 095 TEST_UTIL.shutdownMiniCluster(); 096 } 097 098 /** 099 * Test increment result when there are duplicate rpc request. 100 */ 101 @Test 102 public void testDuplicateIncrement() throws Exception { 103 HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName())); 104 Map<String, String> kvs = new HashMap<>(); 105 kvs.put(HConnectionTestingUtility.SleepAtFirstRpcCall.SLEEP_TIME_CONF_KEY, "2000"); 106 hdt.addCoprocessor(HConnectionTestingUtility.SleepAtFirstRpcCall.class.getName(), null, 1, kvs); 107 TEST_UTIL.createTable(hdt, new byte[][] { ROW }).close(); 108 109 Configuration c = new Configuration(TEST_UTIL.getConfiguration()); 110 c.setInt(HConstants.HBASE_CLIENT_PAUSE, 50); 111 // Client will retry beacuse rpc timeout is small than the sleep time of first rpc call 112 c.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500); 113 114 Connection connection = ConnectionFactory.createConnection(c); 115 Table t = connection.getTable(TableName.valueOf(name.getMethodName())); 116 if (t instanceof HTable) { 117 HTable table = (HTable) t; 118 table.setOperationTimeout(3 * 1000); 119 120 try { 121 Increment inc = new Increment(ROW); 122 inc.addColumn(TEST_UTIL.fam1, QUALIFIER, 1); 123 Result result = table.increment(inc); 124 125 Cell [] cells = result.rawCells(); 126 assertEquals(1, cells.length); 127 assertIncrementKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, 1); 128 129 // Verify expected result 130 Result readResult = table.get(new Get(ROW)); 131 cells = readResult.rawCells(); 132 assertEquals(1, cells.length); 133 assertIncrementKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, 1); 134 } finally { 135 table.close(); 136 connection.close(); 137 } 138 } 139 } 140 141 @Test 142 public void testIncrementWithDeletes() throws Exception { 143 LOG.info("Starting " + this.name.getMethodName()); 144 final TableName TABLENAME = 145 TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName())); 146 Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY); 147 final byte[] COLUMN = Bytes.toBytes("column"); 148 149 ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5); 150 TEST_UTIL.flush(TABLENAME); 151 152 Delete del = new Delete(ROW); 153 ht.delete(del); 154 155 ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5); 156 157 Get get = new Get(ROW); 158 Result r = ht.get(get); 159 assertEquals(1, r.size()); 160 assertEquals(5, Bytes.toLong(r.getValue(FAMILY, COLUMN))); 161 } 162 163 @Test 164 public void testIncrementingInvalidValue() throws Exception { 165 LOG.info("Starting " + this.name.getMethodName()); 166 final TableName TABLENAME = 167 TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName())); 168 Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY); 169 final byte[] COLUMN = Bytes.toBytes("column"); 170 Put p = new Put(ROW); 171 // write an integer here (not a Long) 172 p.addColumn(FAMILY, COLUMN, Bytes.toBytes(5)); 173 ht.put(p); 174 try { 175 ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5); 176 fail("Should have thrown DoNotRetryIOException"); 177 } catch (DoNotRetryIOException iox) { 178 // success 179 } 180 Increment inc = new Increment(ROW); 181 inc.addColumn(FAMILY, COLUMN, 5); 182 try { 183 ht.increment(inc); 184 fail("Should have thrown DoNotRetryIOException"); 185 } catch (DoNotRetryIOException iox) { 186 // success 187 } 188 } 189 190 @Test 191 public void testBatchIncrementsWithReturnResultFalse() throws Exception { 192 LOG.info("Starting testBatchIncrementsWithReturnResultFalse"); 193 final TableName tableName = TableName.valueOf(name.getMethodName()); 194 Table table = TEST_UTIL.createTable(tableName, FAMILY); 195 Increment inc1 = new Increment(Bytes.toBytes("row2")); 196 inc1.setReturnResults(false); 197 inc1.addColumn(FAMILY, Bytes.toBytes("f1"), 1); 198 Increment inc2 = new Increment(Bytes.toBytes("row2")); 199 inc2.setReturnResults(false); 200 inc2.addColumn(FAMILY, Bytes.toBytes("f1"), 1); 201 List<Increment> incs = new ArrayList<>(); 202 incs.add(inc1); 203 incs.add(inc2); 204 Object[] results = new Object[2]; 205 table.batch(incs, results); 206 assertTrue(results.length == 2); 207 for(Object r : results) { 208 Result result = (Result)r; 209 assertTrue(result.isEmpty()); 210 } 211 table.close(); 212 } 213 214 @Test 215 public void testIncrementInvalidArguments() throws Exception { 216 LOG.info("Starting " + this.name.getMethodName()); 217 final TableName TABLENAME = 218 TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName())); 219 Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY); 220 final byte[] COLUMN = Bytes.toBytes("column"); 221 try { 222 // try null row 223 ht.incrementColumnValue(null, FAMILY, COLUMN, 5); 224 fail("Should have thrown IOException"); 225 } catch (IOException iox) { 226 // success 227 } 228 try { 229 // try null family 230 ht.incrementColumnValue(ROW, null, COLUMN, 5); 231 fail("Should have thrown IOException"); 232 } catch (IOException iox) { 233 // success 234 } 235 // try null row 236 try { 237 Increment incNoRow = new Increment((byte [])null); 238 incNoRow.addColumn(FAMILY, COLUMN, 5); 239 fail("Should have thrown IllegalArgumentException"); 240 } catch (IllegalArgumentException iax) { 241 // success 242 } catch (NullPointerException npe) { 243 // success 244 } 245 // try null family 246 try { 247 Increment incNoFamily = new Increment(ROW); 248 incNoFamily.addColumn(null, COLUMN, 5); 249 fail("Should have thrown IllegalArgumentException"); 250 } catch (IllegalArgumentException iax) { 251 // success 252 } 253 } 254 255 @Test 256 public void testIncrementOutOfOrder() throws Exception { 257 LOG.info("Starting " + this.name.getMethodName()); 258 final TableName TABLENAME = 259 TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName())); 260 Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY); 261 262 byte [][] QUALIFIERS = new byte [][] { 263 Bytes.toBytes("B"), Bytes.toBytes("A"), Bytes.toBytes("C") 264 }; 265 266 Increment inc = new Increment(ROW); 267 for (int i=0; i<QUALIFIERS.length; i++) { 268 inc.addColumn(FAMILY, QUALIFIERS[i], 1); 269 } 270 ht.increment(inc); 271 272 // Verify expected results 273 Get get = new Get(ROW); 274 Result r = ht.get(get); 275 Cell [] kvs = r.rawCells(); 276 assertEquals(3, kvs.length); 277 assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 1); 278 assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 1); 279 assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1); 280 281 // Now try multiple columns again 282 inc = new Increment(ROW); 283 for (int i=0; i<QUALIFIERS.length; i++) { 284 inc.addColumn(FAMILY, QUALIFIERS[i], 1); 285 } 286 ht.increment(inc); 287 288 // Verify 289 r = ht.get(get); 290 kvs = r.rawCells(); 291 assertEquals(3, kvs.length); 292 assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 2); 293 assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 2); 294 assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2); 295 } 296 297 @Test 298 public void testIncrementOnSameColumn() throws Exception { 299 LOG.info("Starting " + this.name.getMethodName()); 300 final byte[] TABLENAME = Bytes.toBytes(filterStringSoTableNameSafe(this.name.getMethodName())); 301 Table ht = TEST_UTIL.createTable(TableName.valueOf(TABLENAME), FAMILY); 302 303 byte[][] QUALIFIERS = 304 new byte[][] { Bytes.toBytes("A"), Bytes.toBytes("B"), Bytes.toBytes("C") }; 305 306 Increment inc = new Increment(ROW); 307 for (int i = 0; i < QUALIFIERS.length; i++) { 308 inc.addColumn(FAMILY, QUALIFIERS[i], 1); 309 inc.addColumn(FAMILY, QUALIFIERS[i], 1); 310 } 311 ht.increment(inc); 312 313 // Verify expected results 314 Get get = new Get(ROW); 315 Result r = ht.get(get); 316 Cell[] kvs = r.rawCells(); 317 assertEquals(3, kvs.length); 318 assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1); 319 assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 1); 320 assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1); 321 322 // Now try multiple columns again 323 inc = new Increment(ROW); 324 for (int i = 0; i < QUALIFIERS.length; i++) { 325 inc.addColumn(FAMILY, QUALIFIERS[i], 1); 326 inc.addColumn(FAMILY, QUALIFIERS[i], 1); 327 } 328 ht.increment(inc); 329 330 // Verify 331 r = ht.get(get); 332 kvs = r.rawCells(); 333 assertEquals(3, kvs.length); 334 assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 2); 335 assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 2); 336 assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2); 337 338 ht.close(); 339 } 340 341 @Test 342 public void testIncrementIncrZeroAtFirst() throws Exception { 343 LOG.info("Starting " + this.name.getMethodName()); 344 final TableName TABLENAME = 345 TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName())); 346 Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY); 347 348 byte[] col1 = Bytes.toBytes("col1"); 349 byte[] col2 = Bytes.toBytes("col2"); 350 byte[] col3 = Bytes.toBytes("col3"); 351 352 // Now increment zero at first time incr 353 Increment inc = new Increment(ROW); 354 inc.addColumn(FAMILY, col1, 0); 355 ht.increment(inc); 356 357 // Verify expected results 358 Get get = new Get(ROW); 359 Result r = ht.get(get); 360 Cell [] kvs = r.rawCells(); 361 assertEquals(1, kvs.length); 362 assertNotNull(kvs[0]); 363 assertIncrementKey(kvs[0], ROW, FAMILY, col1, 0); 364 365 // Now try multiple columns by different amounts 366 inc = new Increment(ROW); 367 inc.addColumn(FAMILY, col1, 1); 368 inc.addColumn(FAMILY, col2, 0); 369 inc.addColumn(FAMILY, col3, 2); 370 ht.increment(inc); 371 // Verify 372 get = new Get(ROW); 373 r = ht.get(get); 374 kvs = r.rawCells(); 375 assertEquals(3, kvs.length); 376 assertNotNull(kvs[0]); 377 assertNotNull(kvs[1]); 378 assertNotNull(kvs[2]); 379 assertIncrementKey(kvs[0], ROW, FAMILY, col1, 1); 380 assertIncrementKey(kvs[1], ROW, FAMILY, col2, 0); 381 assertIncrementKey(kvs[2], ROW, FAMILY, col3, 2); 382 } 383 384 @Test 385 public void testIncrement() throws Exception { 386 LOG.info("Starting " + this.name.getMethodName()); 387 final TableName TABLENAME = 388 TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName())); 389 Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY); 390 391 byte [][] ROWS = new byte [][] { 392 Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"), 393 Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"), 394 Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i") 395 }; 396 byte [][] QUALIFIERS = new byte [][] { 397 Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"), 398 Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"), 399 Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i") 400 }; 401 402 // Do some simple single-column increments 403 404 // First with old API 405 ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[0], 1); 406 ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[1], 2); 407 ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[2], 3); 408 ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[3], 4); 409 410 // Now increment things incremented with old and do some new 411 Increment inc = new Increment(ROW); 412 inc.addColumn(FAMILY, QUALIFIERS[1], 1); 413 inc.addColumn(FAMILY, QUALIFIERS[3], 1); 414 inc.addColumn(FAMILY, QUALIFIERS[4], 1); 415 ht.increment(inc); 416 417 // Verify expected results 418 Get get = new Get(ROW); 419 Result r = ht.get(get); 420 Cell [] kvs = r.rawCells(); 421 assertEquals(5, kvs.length); 422 assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1); 423 assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 3); 424 assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 3); 425 assertIncrementKey(kvs[3], ROW, FAMILY, QUALIFIERS[3], 5); 426 assertIncrementKey(kvs[4], ROW, FAMILY, QUALIFIERS[4], 1); 427 428 // Now try multiple columns by different amounts 429 inc = new Increment(ROWS[0]); 430 for (int i=0;i<QUALIFIERS.length;i++) { 431 inc.addColumn(FAMILY, QUALIFIERS[i], i+1); 432 } 433 ht.increment(inc); 434 // Verify 435 get = new Get(ROWS[0]); 436 r = ht.get(get); 437 kvs = r.rawCells(); 438 assertEquals(QUALIFIERS.length, kvs.length); 439 for (int i=0;i<QUALIFIERS.length;i++) { 440 assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], i+1); 441 } 442 443 // Re-increment them 444 inc = new Increment(ROWS[0]); 445 for (int i=0;i<QUALIFIERS.length;i++) { 446 inc.addColumn(FAMILY, QUALIFIERS[i], i+1); 447 } 448 ht.increment(inc); 449 // Verify 450 r = ht.get(get); 451 kvs = r.rawCells(); 452 assertEquals(QUALIFIERS.length, kvs.length); 453 for (int i=0;i<QUALIFIERS.length;i++) { 454 assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1)); 455 } 456 457 // Verify that an Increment of an amount of zero, returns current count; i.e. same as for above 458 // test, that is: 2 * (i + 1). 459 inc = new Increment(ROWS[0]); 460 for (int i = 0; i < QUALIFIERS.length; i++) { 461 inc.addColumn(FAMILY, QUALIFIERS[i], 0); 462 } 463 ht.increment(inc); 464 r = ht.get(get); 465 kvs = r.rawCells(); 466 assertEquals(QUALIFIERS.length, kvs.length); 467 for (int i = 0; i < QUALIFIERS.length; i++) { 468 assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1)); 469 } 470 } 471 472 @Test 473 public void testIncrementWithCustomTimestamp() throws IOException { 474 TableName TABLENAME = TableName.valueOf(name.getMethodName()); 475 Table table = TEST_UTIL.createTable(TABLENAME, FAMILY); 476 long timestamp = 999; 477 Increment increment = new Increment(ROW); 478 increment.add(CellUtil.createCell(ROW, FAMILY, QUALIFIER, timestamp, KeyValue.Type.Put.getCode(), Bytes.toBytes(100L))); 479 Result r = table.increment(increment); 480 assertEquals(1, r.size()); 481 assertEquals(timestamp, r.rawCells()[0].getTimestamp()); 482 r = table.get(new Get(ROW)); 483 assertEquals(1, r.size()); 484 assertEquals(timestamp, r.rawCells()[0].getTimestamp()); 485 r = table.increment(increment); 486 assertEquals(1, r.size()); 487 assertNotEquals(timestamp, r.rawCells()[0].getTimestamp()); 488 r = table.get(new Get(ROW)); 489 assertEquals(1, r.size()); 490 assertNotEquals(timestamp, r.rawCells()[0].getTimestamp()); 491 } 492 493 /** 494 * Call over to the adjacent class's method of same name. 495 */ 496 static void assertIncrementKey(Cell key, byte [] row, byte [] family, 497 byte [] qualifier, long value) throws Exception { 498 TestFromClientSide.assertIncrementKey(key, row, family, qualifier, value); 499 } 500 501 public static String filterStringSoTableNameSafe(final String str) { 502 return str.replaceAll("\\[fast\\=(.*)\\]", ".FAST.is.$1"); 503 } 504}