001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.util.ArrayList;
028import java.util.HashMap;
029import java.util.List;
030import java.util.Map;
031import org.apache.hadoop.conf.Configuration;
032import org.apache.hadoop.hbase.Cell;
033import org.apache.hadoop.hbase.CellUtil;
034import org.apache.hadoop.hbase.DoNotRetryIOException;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.HBaseTestingUtility;
037import org.apache.hadoop.hbase.HConstants;
038import org.apache.hadoop.hbase.HTableDescriptor;
039import org.apache.hadoop.hbase.KeyValue;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
042import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
043import org.apache.hadoop.hbase.testclassification.LargeTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.junit.AfterClass;
046import org.junit.BeforeClass;
047import org.junit.ClassRule;
048import org.junit.Rule;
049import org.junit.Test;
050import org.junit.experimental.categories.Category;
051import org.junit.rules.TestName;
052import org.slf4j.Logger;
053import org.slf4j.LoggerFactory;
054
055/**
056 * Run Increment tests that use the HBase clients; {@link HTable}.
057 *
058 * Test is parameterized to run the slow and fast increment code paths. If fast, in the @before, we
059 * do a rolling restart of the single regionserver so that it can pick up the go fast configuration.
060 * Doing it this way should be faster than starting/stopping a cluster per test.
061 *
062 * Test takes a long time because spin up a cluster between each run -- ugh.
063 */
064@Category(LargeTests.class)
065public class TestIncrementsFromClientSide {
066
067  @ClassRule
068  public static final HBaseClassTestRule CLASS_RULE =
069      HBaseClassTestRule.forClass(TestIncrementsFromClientSide.class);
070
071  final Logger LOG = LoggerFactory.getLogger(getClass());
072  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
073  private static byte [] ROW = Bytes.toBytes("testRow");
074  private static byte [] FAMILY = Bytes.toBytes("testFamily");
075  private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
076  // This test depends on there being only one slave running at at a time. See the @Before
077  // method where we do rolling restart.
078  protected static int SLAVES = 1;
079  @Rule public TestName name = new TestName();
080
081  @BeforeClass
082  public static void beforeClass() throws Exception {
083    Configuration conf = TEST_UTIL.getConfiguration();
084    conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
085        MultiRowMutationEndpoint.class.getName());
086    conf.setBoolean("hbase.table.sanity.checks", true); // enable for below tests
087    // We need more than one region server in this test
088    TEST_UTIL.startMiniCluster(SLAVES);
089  }
090
091  /**
092   * @throws java.lang.Exception
093   */
094  @AfterClass
095  public static void afterClass() throws Exception {
096    TEST_UTIL.shutdownMiniCluster();
097  }
098
099  /**
100   * Test increment result when there are duplicate rpc request.
101   */
102  @Test
103  public void testDuplicateIncrement() throws Exception {
104    HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()));
105    Map<String, String> kvs = new HashMap<>();
106    kvs.put(HConnectionTestingUtility.SleepAtFirstRpcCall.SLEEP_TIME_CONF_KEY, "2000");
107    hdt.addCoprocessor(HConnectionTestingUtility.SleepAtFirstRpcCall.class.getName(), null, 1, kvs);
108    TEST_UTIL.createTable(hdt, new byte[][] { ROW }).close();
109
110    Configuration c = new Configuration(TEST_UTIL.getConfiguration());
111    c.setInt(HConstants.HBASE_CLIENT_PAUSE, 50);
112    // Client will retry beacuse rpc timeout is small than the sleep time of first rpc call
113    c.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500);
114
115    Connection connection = ConnectionFactory.createConnection(c);
116    Table t = connection.getTable(TableName.valueOf(name.getMethodName()));
117    if (t instanceof HTable) {
118      HTable table = (HTable) t;
119      table.setOperationTimeout(3 * 1000);
120
121      try {
122        Increment inc = new Increment(ROW);
123        inc.addColumn(TEST_UTIL.fam1, QUALIFIER, 1);
124        Result result = table.increment(inc);
125
126        Cell [] cells = result.rawCells();
127        assertEquals(1, cells.length);
128        assertIncrementKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, 1);
129
130        // Verify expected result
131        Result readResult = table.get(new Get(ROW));
132        cells = readResult.rawCells();
133        assertEquals(1, cells.length);
134        assertIncrementKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, 1);
135      } finally {
136        table.close();
137        connection.close();
138      }
139    }
140  }
141
142  @Test
143  public void testIncrementWithDeletes() throws Exception {
144    LOG.info("Starting " + this.name.getMethodName());
145    final TableName TABLENAME =
146        TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
147    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
148    final byte[] COLUMN = Bytes.toBytes("column");
149
150    ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
151    TEST_UTIL.flush(TABLENAME);
152
153    Delete del = new Delete(ROW);
154    ht.delete(del);
155
156    ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
157
158    Get get = new Get(ROW);
159    Result r = ht.get(get);
160    assertEquals(1, r.size());
161    assertEquals(5, Bytes.toLong(r.getValue(FAMILY, COLUMN)));
162  }
163
164  @Test
165  public void testIncrementingInvalidValue() throws Exception {
166    LOG.info("Starting " + this.name.getMethodName());
167    final TableName TABLENAME =
168        TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
169    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
170    final byte[] COLUMN = Bytes.toBytes("column");
171    Put p = new Put(ROW);
172    // write an integer here (not a Long)
173    p.addColumn(FAMILY, COLUMN, Bytes.toBytes(5));
174    ht.put(p);
175    try {
176      ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
177      fail("Should have thrown DoNotRetryIOException");
178    } catch (DoNotRetryIOException iox) {
179      // success
180    }
181    Increment inc = new Increment(ROW);
182    inc.addColumn(FAMILY, COLUMN, 5);
183    try {
184      ht.increment(inc);
185      fail("Should have thrown DoNotRetryIOException");
186    } catch (DoNotRetryIOException iox) {
187      // success
188    }
189  }
190
191  @Test
192  public void testBatchIncrementsWithReturnResultFalse() throws Exception {
193    LOG.info("Starting testBatchIncrementsWithReturnResultFalse");
194    final TableName tableName = TableName.valueOf(name.getMethodName());
195    Table table = TEST_UTIL.createTable(tableName, FAMILY);
196    Increment inc1 = new Increment(Bytes.toBytes("row2"));
197    inc1.setReturnResults(false);
198    inc1.addColumn(FAMILY, Bytes.toBytes("f1"), 1);
199    Increment inc2 = new Increment(Bytes.toBytes("row2"));
200    inc2.setReturnResults(false);
201    inc2.addColumn(FAMILY, Bytes.toBytes("f1"), 1);
202    List<Increment> incs = new ArrayList<>();
203    incs.add(inc1);
204    incs.add(inc2);
205    Object[] results = new Object[2];
206    table.batch(incs, results);
207    assertTrue(results.length == 2);
208    for(Object r : results) {
209      Result result = (Result)r;
210      assertTrue(result.isEmpty());
211    }
212    table.close();
213  }
214
215  @Test
216  public void testIncrementInvalidArguments() throws Exception {
217    LOG.info("Starting " + this.name.getMethodName());
218    final TableName TABLENAME =
219        TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
220    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
221    final byte[] COLUMN = Bytes.toBytes("column");
222    try {
223      // try null row
224      ht.incrementColumnValue(null, FAMILY, COLUMN, 5);
225      fail("Should have thrown IOException");
226    } catch (IOException iox) {
227      // success
228    }
229    try {
230      // try null family
231      ht.incrementColumnValue(ROW, null, COLUMN, 5);
232      fail("Should have thrown IOException");
233    } catch (IOException iox) {
234      // success
235    }
236    // try null row
237    try {
238      Increment incNoRow = new Increment((byte [])null);
239      incNoRow.addColumn(FAMILY, COLUMN, 5);
240      fail("Should have thrown IllegalArgumentException");
241    } catch (IllegalArgumentException iax) {
242      // success
243    } catch (NullPointerException npe) {
244      // success
245    }
246    // try null family
247    try {
248      Increment incNoFamily = new Increment(ROW);
249      incNoFamily.addColumn(null, COLUMN, 5);
250      fail("Should have thrown IllegalArgumentException");
251    } catch (IllegalArgumentException iax) {
252      // success
253    }
254  }
255
256  @Test
257  public void testIncrementOutOfOrder() throws Exception {
258    LOG.info("Starting " + this.name.getMethodName());
259    final TableName TABLENAME =
260        TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
261    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
262
263    byte [][] QUALIFIERS = new byte [][] {
264      Bytes.toBytes("B"), Bytes.toBytes("A"), Bytes.toBytes("C")
265    };
266
267    Increment inc = new Increment(ROW);
268    for (int i=0; i<QUALIFIERS.length; i++) {
269      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
270    }
271    ht.increment(inc);
272
273    // Verify expected results
274    Get get = new Get(ROW);
275    Result r = ht.get(get);
276    Cell [] kvs = r.rawCells();
277    assertEquals(3, kvs.length);
278    assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 1);
279    assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 1);
280    assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1);
281
282    // Now try multiple columns again
283    inc = new Increment(ROW);
284    for (int i=0; i<QUALIFIERS.length; i++) {
285      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
286    }
287    ht.increment(inc);
288
289    // Verify
290    r = ht.get(get);
291    kvs = r.rawCells();
292    assertEquals(3, kvs.length);
293    assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 2);
294    assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 2);
295    assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2);
296  }
297
298  @Test
299  public void testIncrementOnSameColumn() throws Exception {
300    LOG.info("Starting " + this.name.getMethodName());
301    final byte[] TABLENAME = Bytes.toBytes(filterStringSoTableNameSafe(this.name.getMethodName()));
302    Table ht = TEST_UTIL.createTable(TableName.valueOf(TABLENAME), FAMILY);
303
304    byte[][] QUALIFIERS =
305        new byte[][] { Bytes.toBytes("A"), Bytes.toBytes("B"), Bytes.toBytes("C") };
306
307    Increment inc = new Increment(ROW);
308    for (int i = 0; i < QUALIFIERS.length; i++) {
309      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
310      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
311    }
312    ht.increment(inc);
313
314    // Verify expected results
315    Get get = new Get(ROW);
316    Result r = ht.get(get);
317    Cell[] kvs = r.rawCells();
318    assertEquals(3, kvs.length);
319    assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1);
320    assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 1);
321    assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1);
322
323    // Now try multiple columns again
324    inc = new Increment(ROW);
325    for (int i = 0; i < QUALIFIERS.length; i++) {
326      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
327      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
328    }
329    ht.increment(inc);
330
331    // Verify
332    r = ht.get(get);
333    kvs = r.rawCells();
334    assertEquals(3, kvs.length);
335    assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 2);
336    assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 2);
337    assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2);
338
339    ht.close();
340  }
341
342  @Test
343  public void testIncrementIncrZeroAtFirst() throws Exception {
344    LOG.info("Starting " + this.name.getMethodName());
345    final TableName TABLENAME =
346            TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
347    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
348
349    byte[] col1 = Bytes.toBytes("col1");
350    byte[] col2 = Bytes.toBytes("col2");
351    byte[] col3 = Bytes.toBytes("col3");
352
353    // Now increment zero at first time incr
354    Increment inc = new Increment(ROW);
355    inc.addColumn(FAMILY, col1, 0);
356    ht.increment(inc);
357
358    // Verify expected results
359    Get get = new Get(ROW);
360    Result r = ht.get(get);
361    Cell [] kvs = r.rawCells();
362    assertEquals(1, kvs.length);
363    assertNotNull(kvs[0]);
364    assertIncrementKey(kvs[0], ROW, FAMILY, col1, 0);
365
366    // Now try multiple columns by different amounts
367    inc = new Increment(ROW);
368    inc.addColumn(FAMILY, col1, 1);
369    inc.addColumn(FAMILY, col2, 0);
370    inc.addColumn(FAMILY, col3, 2);
371    ht.increment(inc);
372    // Verify
373    get = new Get(ROW);
374    r = ht.get(get);
375    kvs = r.rawCells();
376    assertEquals(3, kvs.length);
377    assertNotNull(kvs[0]);
378    assertNotNull(kvs[1]);
379    assertNotNull(kvs[2]);
380    assertIncrementKey(kvs[0], ROW, FAMILY, col1, 1);
381    assertIncrementKey(kvs[1], ROW, FAMILY, col2, 0);
382    assertIncrementKey(kvs[2], ROW, FAMILY, col3, 2);
383  }
384
385  @Test
386  public void testIncrement() throws Exception {
387    LOG.info("Starting " + this.name.getMethodName());
388    final TableName TABLENAME =
389        TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
390    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
391
392    byte [][] ROWS = new byte [][] {
393        Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
394        Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
395        Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
396    };
397    byte [][] QUALIFIERS = new byte [][] {
398        Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
399        Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
400        Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
401    };
402
403    // Do some simple single-column increments
404
405    // First with old API
406    ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[0], 1);
407    ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[1], 2);
408    ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[2], 3);
409    ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[3], 4);
410
411    // Now increment things incremented with old and do some new
412    Increment inc = new Increment(ROW);
413    inc.addColumn(FAMILY, QUALIFIERS[1], 1);
414    inc.addColumn(FAMILY, QUALIFIERS[3], 1);
415    inc.addColumn(FAMILY, QUALIFIERS[4], 1);
416    ht.increment(inc);
417
418    // Verify expected results
419    Get get = new Get(ROW);
420    Result r = ht.get(get);
421    Cell [] kvs = r.rawCells();
422    assertEquals(5, kvs.length);
423    assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1);
424    assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 3);
425    assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 3);
426    assertIncrementKey(kvs[3], ROW, FAMILY, QUALIFIERS[3], 5);
427    assertIncrementKey(kvs[4], ROW, FAMILY, QUALIFIERS[4], 1);
428
429    // Now try multiple columns by different amounts
430    inc = new Increment(ROWS[0]);
431    for (int i=0;i<QUALIFIERS.length;i++) {
432      inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
433    }
434    ht.increment(inc);
435    // Verify
436    get = new Get(ROWS[0]);
437    r = ht.get(get);
438    kvs = r.rawCells();
439    assertEquals(QUALIFIERS.length, kvs.length);
440    for (int i=0;i<QUALIFIERS.length;i++) {
441      assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], i+1);
442    }
443
444    // Re-increment them
445    inc = new Increment(ROWS[0]);
446    for (int i=0;i<QUALIFIERS.length;i++) {
447      inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
448    }
449    ht.increment(inc);
450    // Verify
451    r = ht.get(get);
452    kvs = r.rawCells();
453    assertEquals(QUALIFIERS.length, kvs.length);
454    for (int i=0;i<QUALIFIERS.length;i++) {
455      assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1));
456    }
457
458    // Verify that an Increment of an amount of zero, returns current count; i.e. same as for above
459    // test, that is: 2 * (i + 1).
460    inc = new Increment(ROWS[0]);
461    for (int i = 0; i < QUALIFIERS.length; i++) {
462      inc.addColumn(FAMILY, QUALIFIERS[i], 0);
463    }
464    ht.increment(inc);
465    r = ht.get(get);
466    kvs = r.rawCells();
467    assertEquals(QUALIFIERS.length, kvs.length);
468    for (int i = 0; i < QUALIFIERS.length; i++) {
469      assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1));
470    }
471  }
472
473  @Test
474  public void testIncrementWithCustomTimestamp() throws IOException {
475    TableName TABLENAME = TableName.valueOf(name.getMethodName());
476    Table table = TEST_UTIL.createTable(TABLENAME, FAMILY);
477    long timestamp = 999;
478    Increment increment = new Increment(ROW);
479    increment.add(CellUtil.createCell(ROW, FAMILY, QUALIFIER, timestamp, KeyValue.Type.Put.getCode(), Bytes.toBytes(100L)));
480    Result r = table.increment(increment);
481    assertEquals(1, r.size());
482    assertEquals(timestamp, r.rawCells()[0].getTimestamp());
483    r = table.get(new Get(ROW));
484    assertEquals(1, r.size());
485    assertEquals(timestamp, r.rawCells()[0].getTimestamp());
486    r = table.increment(increment);
487    assertEquals(1, r.size());
488    assertNotEquals(timestamp, r.rawCells()[0].getTimestamp());
489    r = table.get(new Get(ROW));
490    assertEquals(1, r.size());
491    assertNotEquals(timestamp, r.rawCells()[0].getTimestamp());
492  }
493
494  /**
495   * Call over to the adjacent class's method of same name.
496   */
497  static void assertIncrementKey(Cell key, byte [] row, byte [] family,
498      byte [] qualifier, long value) throws Exception {
499    TestFromClientSide.assertIncrementKey(key, row, family, qualifier, value);
500  }
501
502  public static String filterStringSoTableNameSafe(final String str) {
503    return str.replaceAll("\\[fast\\=(.*)\\]", ".FAST.is.$1");
504  }
505}