001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.util.ArrayList;
028import java.util.Collections;
029import java.util.HashMap;
030import java.util.List;
031import java.util.Map;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.hbase.Cell;
034import org.apache.hadoop.hbase.CellUtil;
035import org.apache.hadoop.hbase.DoNotRetryIOException;
036import org.apache.hadoop.hbase.HBaseClassTestRule;
037import org.apache.hadoop.hbase.HBaseTestingUtility;
038import org.apache.hadoop.hbase.HConstants;
039import org.apache.hadoop.hbase.HTableDescriptor;
040import org.apache.hadoop.hbase.KeyValue;
041import org.apache.hadoop.hbase.TableName;
042import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
043import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
044import org.apache.hadoop.hbase.testclassification.LargeTests;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.junit.AfterClass;
047import org.junit.BeforeClass;
048import org.junit.ClassRule;
049import org.junit.Rule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052import org.junit.rules.TestName;
053import org.slf4j.Logger;
054import org.slf4j.LoggerFactory;
055
056/**
057 * Run Increment tests that use the HBase clients; {@link TableBuilder}.
058 *
059 * Test is parameterized to run the slow and fast increment code paths. If fast, in the @before, we
060 * do a rolling restart of the single regionserver so that it can pick up the go fast configuration.
061 * Doing it this way should be faster than starting/stopping a cluster per test.
062 *
063 * Test takes a long time because spin up a cluster between each run -- ugh.
064 */
065@Category(LargeTests.class)
066public class TestIncrementsFromClientSide {
067
068  @ClassRule
069  public static final HBaseClassTestRule CLASS_RULE =
070      HBaseClassTestRule.forClass(TestIncrementsFromClientSide.class);
071
072  final Logger LOG = LoggerFactory.getLogger(getClass());
073  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
074  private static byte [] ROW = Bytes.toBytes("testRow");
075  private static byte [] FAMILY = Bytes.toBytes("testFamily");
076  private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
077  // This test depends on there being only one slave running at at a time. See the @Before
078  // method where we do rolling restart.
079  protected static int SLAVES = 1;
080  @Rule public TestName name = new TestName();
081
082  @BeforeClass
083  public static void beforeClass() throws Exception {
084    Configuration conf = TEST_UTIL.getConfiguration();
085    conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
086        MultiRowMutationEndpoint.class.getName());
087    // We need more than one region server in this test
088    TEST_UTIL.startMiniCluster(SLAVES);
089  }
090
091  /**
092   * @throws java.lang.Exception
093   */
094  @AfterClass
095  public static void afterClass() throws Exception {
096    TEST_UTIL.shutdownMiniCluster();
097  }
098
099  /**
100   * Test increment result when there are duplicate rpc request.
101   */
102  @Test
103  public void testDuplicateIncrement() throws Exception {
104    HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()));
105    Map<String, String> kvs = new HashMap<>();
106    kvs.put(HConnectionTestingUtility.SleepAtFirstRpcCall.SLEEP_TIME_CONF_KEY, "2000");
107    hdt.addCoprocessor(HConnectionTestingUtility.SleepAtFirstRpcCall.class.getName(), null, 1, kvs);
108    TEST_UTIL.createTable(hdt, new byte[][] { ROW }).close();
109
110    Configuration c = new Configuration(TEST_UTIL.getConfiguration());
111    c.setInt(HConstants.HBASE_CLIENT_PAUSE, 50);
112    // Client will retry beacuse rpc timeout is small than the sleep time of first rpc call
113    c.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500);
114
115    Connection connection = ConnectionFactory.createConnection(c);
116    Table t = connection.getTable(TableName.valueOf(name.getMethodName()));
117    if (t instanceof HTable) {
118      HTable table = (HTable) t;
119      table.setOperationTimeout(3 * 1000);
120
121      try {
122        Increment inc = new Increment(ROW);
123        inc.addColumn(TEST_UTIL.fam1, QUALIFIER, 1);
124        Result result = table.increment(inc);
125
126        Cell [] cells = result.rawCells();
127        assertEquals(1, cells.length);
128        assertIncrementKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, 1);
129
130        // Verify expected result
131        Result readResult = table.get(new Get(ROW));
132        cells = readResult.rawCells();
133        assertEquals(1, cells.length);
134        assertIncrementKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, 1);
135      } finally {
136        table.close();
137        connection.close();
138      }
139    }
140  }
141
142  /**
143   * Test batch increment result when there are duplicate rpc request.
144   */
145  @Test
146  public void testDuplicateBatchIncrement() throws Exception {
147    HTableDescriptor hdt =
148      TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()));
149    Map<String, String> kvs = new HashMap<>();
150    kvs.put(HConnectionTestingUtility.SleepAtFirstRpcCall.SLEEP_TIME_CONF_KEY, "2000");
151    hdt.addCoprocessor(HConnectionTestingUtility.SleepAtFirstRpcCall.class.getName(), null, 1,
152      kvs);
153    TEST_UTIL.createTable(hdt, new byte[][] { ROW }).close();
154
155    Configuration c = new Configuration(TEST_UTIL.getConfiguration());
156    c.setInt(HConstants.HBASE_CLIENT_PAUSE, 50);
157    // Client will retry beacuse rpc timeout is small than the sleep time of first rpc call
158    c.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500);
159
160    try (Connection connection = ConnectionFactory.createConnection(c);
161      Table table = connection.getTableBuilder(TableName.valueOf(name.getMethodName()), null)
162        .setOperationTimeout(3 * 1000).build()) {
163      Increment inc = new Increment(ROW);
164      inc.addColumn(HBaseTestingUtility.fam1, QUALIFIER, 1);
165
166      // Batch increment
167      Object[] results = new Object[1];
168      table.batch(Collections.singletonList(inc), results);
169
170      Cell[] cells = ((Result) results[0]).rawCells();
171      assertEquals(1, cells.length);
172      assertIncrementKey(cells[0], ROW, HBaseTestingUtility.fam1, QUALIFIER, 1);
173
174      // Verify expected result
175      Result readResult = table.get(new Get(ROW));
176      cells = readResult.rawCells();
177      assertEquals(1, cells.length);
178      assertIncrementKey(cells[0], ROW, HBaseTestingUtility.fam1, QUALIFIER, 1);
179    }
180  }
181
182  @Test
183  public void testIncrementWithDeletes() throws Exception {
184    LOG.info("Starting " + this.name.getMethodName());
185    final TableName TABLENAME =
186        TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
187    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
188    final byte[] COLUMN = Bytes.toBytes("column");
189
190    ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
191    TEST_UTIL.flush(TABLENAME);
192
193    Delete del = new Delete(ROW);
194    ht.delete(del);
195
196    ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
197
198    Get get = new Get(ROW);
199    Result r = ht.get(get);
200    assertEquals(1, r.size());
201    assertEquals(5, Bytes.toLong(r.getValue(FAMILY, COLUMN)));
202  }
203
204  @Test
205  public void testIncrementingInvalidValue() throws Exception {
206    LOG.info("Starting " + this.name.getMethodName());
207    final TableName TABLENAME =
208        TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
209    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
210    final byte[] COLUMN = Bytes.toBytes("column");
211    Put p = new Put(ROW);
212    // write an integer here (not a Long)
213    p.addColumn(FAMILY, COLUMN, Bytes.toBytes(5));
214    ht.put(p);
215    try {
216      ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
217      fail("Should have thrown DoNotRetryIOException");
218    } catch (DoNotRetryIOException iox) {
219      // success
220    }
221    Increment inc = new Increment(ROW);
222    inc.addColumn(FAMILY, COLUMN, 5);
223    try {
224      ht.increment(inc);
225      fail("Should have thrown DoNotRetryIOException");
226    } catch (DoNotRetryIOException iox) {
227      // success
228    }
229  }
230
231  @Test
232  public void testBatchIncrementsWithReturnResultFalse() throws Exception {
233    LOG.info("Starting testBatchIncrementsWithReturnResultFalse");
234    final TableName tableName = TableName.valueOf(name.getMethodName());
235    Table table = TEST_UTIL.createTable(tableName, FAMILY);
236    Increment inc1 = new Increment(Bytes.toBytes("row2"));
237    inc1.setReturnResults(false);
238    inc1.addColumn(FAMILY, Bytes.toBytes("f1"), 1);
239    Increment inc2 = new Increment(Bytes.toBytes("row2"));
240    inc2.setReturnResults(false);
241    inc2.addColumn(FAMILY, Bytes.toBytes("f1"), 1);
242    List<Increment> incs = new ArrayList<>();
243    incs.add(inc1);
244    incs.add(inc2);
245    Object[] results = new Object[2];
246    table.batch(incs, results);
247    assertTrue(results.length == 2);
248    for(Object r : results) {
249      Result result = (Result)r;
250      assertTrue(result.isEmpty());
251    }
252    table.close();
253  }
254
255  @Test
256  public void testIncrementInvalidArguments() throws Exception {
257    LOG.info("Starting " + this.name.getMethodName());
258    final TableName TABLENAME =
259        TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
260    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
261    final byte[] COLUMN = Bytes.toBytes("column");
262    try {
263      // try null row
264      ht.incrementColumnValue(null, FAMILY, COLUMN, 5);
265      fail("Should have thrown IOException");
266    } catch (IOException iox) {
267      // success
268    }
269    try {
270      // try null family
271      ht.incrementColumnValue(ROW, null, COLUMN, 5);
272      fail("Should have thrown IOException");
273    } catch (IOException iox) {
274      // success
275    }
276    // try null row
277    try {
278      Increment incNoRow = new Increment((byte [])null);
279      incNoRow.addColumn(FAMILY, COLUMN, 5);
280      fail("Should have thrown IllegalArgumentException");
281    } catch (IllegalArgumentException iax) {
282      // success
283    } catch (NullPointerException npe) {
284      // success
285    }
286    // try null family
287    try {
288      Increment incNoFamily = new Increment(ROW);
289      incNoFamily.addColumn(null, COLUMN, 5);
290      fail("Should have thrown IllegalArgumentException");
291    } catch (IllegalArgumentException iax) {
292      // success
293    }
294  }
295
296  @Test
297  public void testIncrementOutOfOrder() throws Exception {
298    LOG.info("Starting " + this.name.getMethodName());
299    final TableName TABLENAME =
300        TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
301    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
302
303    byte [][] QUALIFIERS = new byte [][] {
304      Bytes.toBytes("B"), Bytes.toBytes("A"), Bytes.toBytes("C")
305    };
306
307    Increment inc = new Increment(ROW);
308    for (int i=0; i<QUALIFIERS.length; i++) {
309      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
310    }
311    ht.increment(inc);
312
313    // Verify expected results
314    Get get = new Get(ROW);
315    Result r = ht.get(get);
316    Cell [] kvs = r.rawCells();
317    assertEquals(3, kvs.length);
318    assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 1);
319    assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 1);
320    assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1);
321
322    // Now try multiple columns again
323    inc = new Increment(ROW);
324    for (int i=0; i<QUALIFIERS.length; i++) {
325      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
326    }
327    ht.increment(inc);
328
329    // Verify
330    r = ht.get(get);
331    kvs = r.rawCells();
332    assertEquals(3, kvs.length);
333    assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 2);
334    assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 2);
335    assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2);
336  }
337
338  @Test
339  public void testIncrementOnSameColumn() throws Exception {
340    LOG.info("Starting " + this.name.getMethodName());
341    final byte[] TABLENAME = Bytes.toBytes(filterStringSoTableNameSafe(this.name.getMethodName()));
342    Table ht = TEST_UTIL.createTable(TableName.valueOf(TABLENAME), FAMILY);
343
344    byte[][] QUALIFIERS =
345        new byte[][] { Bytes.toBytes("A"), Bytes.toBytes("B"), Bytes.toBytes("C") };
346
347    Increment inc = new Increment(ROW);
348    for (int i = 0; i < QUALIFIERS.length; i++) {
349      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
350      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
351    }
352    ht.increment(inc);
353
354    // Verify expected results
355    Get get = new Get(ROW);
356    Result r = ht.get(get);
357    Cell[] kvs = r.rawCells();
358    assertEquals(3, kvs.length);
359    assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1);
360    assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 1);
361    assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1);
362
363    // Now try multiple columns again
364    inc = new Increment(ROW);
365    for (int i = 0; i < QUALIFIERS.length; i++) {
366      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
367      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
368    }
369    ht.increment(inc);
370
371    // Verify
372    r = ht.get(get);
373    kvs = r.rawCells();
374    assertEquals(3, kvs.length);
375    assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 2);
376    assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 2);
377    assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2);
378
379    ht.close();
380  }
381
382  @Test
383  public void testIncrementIncrZeroAtFirst() throws Exception {
384    LOG.info("Starting " + this.name.getMethodName());
385    final TableName TABLENAME =
386            TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
387    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
388
389    byte[] col1 = Bytes.toBytes("col1");
390    byte[] col2 = Bytes.toBytes("col2");
391    byte[] col3 = Bytes.toBytes("col3");
392
393    // Now increment zero at first time incr
394    Increment inc = new Increment(ROW);
395    inc.addColumn(FAMILY, col1, 0);
396    ht.increment(inc);
397
398    // Verify expected results
399    Get get = new Get(ROW);
400    Result r = ht.get(get);
401    Cell [] kvs = r.rawCells();
402    assertEquals(1, kvs.length);
403    assertNotNull(kvs[0]);
404    assertIncrementKey(kvs[0], ROW, FAMILY, col1, 0);
405
406    // Now try multiple columns by different amounts
407    inc = new Increment(ROW);
408    inc.addColumn(FAMILY, col1, 1);
409    inc.addColumn(FAMILY, col2, 0);
410    inc.addColumn(FAMILY, col3, 2);
411    ht.increment(inc);
412    // Verify
413    get = new Get(ROW);
414    r = ht.get(get);
415    kvs = r.rawCells();
416    assertEquals(3, kvs.length);
417    assertNotNull(kvs[0]);
418    assertNotNull(kvs[1]);
419    assertNotNull(kvs[2]);
420    assertIncrementKey(kvs[0], ROW, FAMILY, col1, 1);
421    assertIncrementKey(kvs[1], ROW, FAMILY, col2, 0);
422    assertIncrementKey(kvs[2], ROW, FAMILY, col3, 2);
423  }
424
425  @Test
426  public void testIncrement() throws Exception {
427    LOG.info("Starting " + this.name.getMethodName());
428    final TableName TABLENAME =
429        TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
430    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
431
432    byte [][] ROWS = new byte [][] {
433        Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
434        Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
435        Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
436    };
437    byte [][] QUALIFIERS = new byte [][] {
438        Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
439        Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
440        Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
441    };
442
443    // Do some simple single-column increments
444
445    // First with old API
446    ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[0], 1);
447    ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[1], 2);
448    ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[2], 3);
449    ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[3], 4);
450
451    // Now increment things incremented with old and do some new
452    Increment inc = new Increment(ROW);
453    inc.addColumn(FAMILY, QUALIFIERS[1], 1);
454    inc.addColumn(FAMILY, QUALIFIERS[3], 1);
455    inc.addColumn(FAMILY, QUALIFIERS[4], 1);
456    ht.increment(inc);
457
458    // Verify expected results
459    Get get = new Get(ROW);
460    Result r = ht.get(get);
461    Cell [] kvs = r.rawCells();
462    assertEquals(5, kvs.length);
463    assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1);
464    assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 3);
465    assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 3);
466    assertIncrementKey(kvs[3], ROW, FAMILY, QUALIFIERS[3], 5);
467    assertIncrementKey(kvs[4], ROW, FAMILY, QUALIFIERS[4], 1);
468
469    // Now try multiple columns by different amounts
470    inc = new Increment(ROWS[0]);
471    for (int i=0;i<QUALIFIERS.length;i++) {
472      inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
473    }
474    ht.increment(inc);
475    // Verify
476    get = new Get(ROWS[0]);
477    r = ht.get(get);
478    kvs = r.rawCells();
479    assertEquals(QUALIFIERS.length, kvs.length);
480    for (int i=0;i<QUALIFIERS.length;i++) {
481      assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], i+1);
482    }
483
484    // Re-increment them
485    inc = new Increment(ROWS[0]);
486    for (int i=0;i<QUALIFIERS.length;i++) {
487      inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
488    }
489    ht.increment(inc);
490    // Verify
491    r = ht.get(get);
492    kvs = r.rawCells();
493    assertEquals(QUALIFIERS.length, kvs.length);
494    for (int i=0;i<QUALIFIERS.length;i++) {
495      assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1));
496    }
497
498    // Verify that an Increment of an amount of zero, returns current count; i.e. same as for above
499    // test, that is: 2 * (i + 1).
500    inc = new Increment(ROWS[0]);
501    for (int i = 0; i < QUALIFIERS.length; i++) {
502      inc.addColumn(FAMILY, QUALIFIERS[i], 0);
503    }
504    ht.increment(inc);
505    r = ht.get(get);
506    kvs = r.rawCells();
507    assertEquals(QUALIFIERS.length, kvs.length);
508    for (int i = 0; i < QUALIFIERS.length; i++) {
509      assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1));
510    }
511  }
512
513  @Test
514  public void testIncrementWithCustomTimestamp() throws IOException {
515    TableName TABLENAME = TableName.valueOf(name.getMethodName());
516    Table table = TEST_UTIL.createTable(TABLENAME, FAMILY);
517    long timestamp = 999;
518    Increment increment = new Increment(ROW);
519    increment.add(CellUtil.createCell(ROW, FAMILY, QUALIFIER, timestamp, KeyValue.Type.Put.getCode(), Bytes.toBytes(100L)));
520    Result r = table.increment(increment);
521    assertEquals(1, r.size());
522    assertEquals(timestamp, r.rawCells()[0].getTimestamp());
523    r = table.get(new Get(ROW));
524    assertEquals(1, r.size());
525    assertEquals(timestamp, r.rawCells()[0].getTimestamp());
526    r = table.increment(increment);
527    assertEquals(1, r.size());
528    assertNotEquals(timestamp, r.rawCells()[0].getTimestamp());
529    r = table.get(new Get(ROW));
530    assertEquals(1, r.size());
531    assertNotEquals(timestamp, r.rawCells()[0].getTimestamp());
532  }
533
534  /**
535   * Call over to the adjacent class's method of same name.
536   */
537  static void assertIncrementKey(Cell key, byte [] row, byte [] family,
538      byte [] qualifier, long value) throws Exception {
539    TestFromClientSide.assertIncrementKey(key, row, family, qualifier, value);
540  }
541
542  public static String filterStringSoTableNameSafe(final String str) {
543    return str.replaceAll("\\[fast\\=(.*)\\]", ".FAST.is.$1");
544  }
545}