001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.util.ArrayList;
028import java.util.HashMap;
029import java.util.List;
030import java.util.Map;
031import org.apache.hadoop.conf.Configuration;
032import org.apache.hadoop.hbase.Cell;
033import org.apache.hadoop.hbase.CellUtil;
034import org.apache.hadoop.hbase.DoNotRetryIOException;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.HBaseTestingUtility;
037import org.apache.hadoop.hbase.HConstants;
038import org.apache.hadoop.hbase.HTableDescriptor;
039import org.apache.hadoop.hbase.KeyValue;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
042import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
043import org.apache.hadoop.hbase.testclassification.LargeTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.junit.AfterClass;
046import org.junit.BeforeClass;
047import org.junit.ClassRule;
048import org.junit.Rule;
049import org.junit.Test;
050import org.junit.experimental.categories.Category;
051import org.junit.rules.TestName;
052import org.slf4j.Logger;
053import org.slf4j.LoggerFactory;
054
055/**
056 * Run Increment tests that use the HBase clients; {@link HTable}.
057 *
058 * Test is parameterized to run the slow and fast increment code paths. If fast, in the @before, we
059 * do a rolling restart of the single regionserver so that it can pick up the go fast configuration.
060 * Doing it this way should be faster than starting/stopping a cluster per test.
061 *
062 * Test takes a long time because spin up a cluster between each run -- ugh.
063 */
064@Category(LargeTests.class)
065public class TestIncrementsFromClientSide {
066
067  @ClassRule
068  public static final HBaseClassTestRule CLASS_RULE =
069      HBaseClassTestRule.forClass(TestIncrementsFromClientSide.class);
070
071  final Logger LOG = LoggerFactory.getLogger(getClass());
072  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
073  private static byte [] ROW = Bytes.toBytes("testRow");
074  private static byte [] FAMILY = Bytes.toBytes("testFamily");
075  private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
076  // This test depends on there being only one slave running at at a time. See the @Before
077  // method where we do rolling restart.
078  protected static int SLAVES = 1;
079  @Rule public TestName name = new TestName();
080
081  @BeforeClass
082  public static void beforeClass() throws Exception {
083    Configuration conf = TEST_UTIL.getConfiguration();
084    conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
085        MultiRowMutationEndpoint.class.getName());
086    // We need more than one region server in this test
087    TEST_UTIL.startMiniCluster(SLAVES);
088  }
089
090  /**
091   * @throws java.lang.Exception
092   */
093  @AfterClass
094  public static void afterClass() throws Exception {
095    TEST_UTIL.shutdownMiniCluster();
096  }
097
098  /**
099   * Test increment result when there are duplicate rpc request.
100   */
101  @Test
102  public void testDuplicateIncrement() throws Exception {
103    HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()));
104    Map<String, String> kvs = new HashMap<>();
105    kvs.put(HConnectionTestingUtility.SleepAtFirstRpcCall.SLEEP_TIME_CONF_KEY, "2000");
106    hdt.addCoprocessor(HConnectionTestingUtility.SleepAtFirstRpcCall.class.getName(), null, 1, kvs);
107    TEST_UTIL.createTable(hdt, new byte[][] { ROW }).close();
108
109    Configuration c = new Configuration(TEST_UTIL.getConfiguration());
110    c.setInt(HConstants.HBASE_CLIENT_PAUSE, 50);
111    // Client will retry beacuse rpc timeout is small than the sleep time of first rpc call
112    c.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500);
113
114    Connection connection = ConnectionFactory.createConnection(c);
115    Table t = connection.getTable(TableName.valueOf(name.getMethodName()));
116    if (t instanceof HTable) {
117      HTable table = (HTable) t;
118      table.setOperationTimeout(3 * 1000);
119
120      try {
121        Increment inc = new Increment(ROW);
122        inc.addColumn(TEST_UTIL.fam1, QUALIFIER, 1);
123        Result result = table.increment(inc);
124
125        Cell [] cells = result.rawCells();
126        assertEquals(1, cells.length);
127        assertIncrementKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, 1);
128
129        // Verify expected result
130        Result readResult = table.get(new Get(ROW));
131        cells = readResult.rawCells();
132        assertEquals(1, cells.length);
133        assertIncrementKey(cells[0], ROW, TEST_UTIL.fam1, QUALIFIER, 1);
134      } finally {
135        table.close();
136        connection.close();
137      }
138    }
139  }
140
141  @Test
142  public void testIncrementWithDeletes() throws Exception {
143    LOG.info("Starting " + this.name.getMethodName());
144    final TableName TABLENAME =
145        TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
146    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
147    final byte[] COLUMN = Bytes.toBytes("column");
148
149    ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
150    TEST_UTIL.flush(TABLENAME);
151
152    Delete del = new Delete(ROW);
153    ht.delete(del);
154
155    ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
156
157    Get get = new Get(ROW);
158    Result r = ht.get(get);
159    assertEquals(1, r.size());
160    assertEquals(5, Bytes.toLong(r.getValue(FAMILY, COLUMN)));
161  }
162
163  @Test
164  public void testIncrementingInvalidValue() throws Exception {
165    LOG.info("Starting " + this.name.getMethodName());
166    final TableName TABLENAME =
167        TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
168    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
169    final byte[] COLUMN = Bytes.toBytes("column");
170    Put p = new Put(ROW);
171    // write an integer here (not a Long)
172    p.addColumn(FAMILY, COLUMN, Bytes.toBytes(5));
173    ht.put(p);
174    try {
175      ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
176      fail("Should have thrown DoNotRetryIOException");
177    } catch (DoNotRetryIOException iox) {
178      // success
179    }
180    Increment inc = new Increment(ROW);
181    inc.addColumn(FAMILY, COLUMN, 5);
182    try {
183      ht.increment(inc);
184      fail("Should have thrown DoNotRetryIOException");
185    } catch (DoNotRetryIOException iox) {
186      // success
187    }
188  }
189
190  @Test
191  public void testBatchIncrementsWithReturnResultFalse() throws Exception {
192    LOG.info("Starting testBatchIncrementsWithReturnResultFalse");
193    final TableName tableName = TableName.valueOf(name.getMethodName());
194    Table table = TEST_UTIL.createTable(tableName, FAMILY);
195    Increment inc1 = new Increment(Bytes.toBytes("row2"));
196    inc1.setReturnResults(false);
197    inc1.addColumn(FAMILY, Bytes.toBytes("f1"), 1);
198    Increment inc2 = new Increment(Bytes.toBytes("row2"));
199    inc2.setReturnResults(false);
200    inc2.addColumn(FAMILY, Bytes.toBytes("f1"), 1);
201    List<Increment> incs = new ArrayList<>();
202    incs.add(inc1);
203    incs.add(inc2);
204    Object[] results = new Object[2];
205    table.batch(incs, results);
206    assertTrue(results.length == 2);
207    for(Object r : results) {
208      Result result = (Result)r;
209      assertTrue(result.isEmpty());
210    }
211    table.close();
212  }
213
214  @Test
215  public void testIncrementInvalidArguments() throws Exception {
216    LOG.info("Starting " + this.name.getMethodName());
217    final TableName TABLENAME =
218        TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
219    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
220    final byte[] COLUMN = Bytes.toBytes("column");
221    try {
222      // try null row
223      ht.incrementColumnValue(null, FAMILY, COLUMN, 5);
224      fail("Should have thrown IOException");
225    } catch (IOException iox) {
226      // success
227    }
228    try {
229      // try null family
230      ht.incrementColumnValue(ROW, null, COLUMN, 5);
231      fail("Should have thrown IOException");
232    } catch (IOException iox) {
233      // success
234    }
235    // try null row
236    try {
237      Increment incNoRow = new Increment((byte [])null);
238      incNoRow.addColumn(FAMILY, COLUMN, 5);
239      fail("Should have thrown IllegalArgumentException");
240    } catch (IllegalArgumentException iax) {
241      // success
242    } catch (NullPointerException npe) {
243      // success
244    }
245    // try null family
246    try {
247      Increment incNoFamily = new Increment(ROW);
248      incNoFamily.addColumn(null, COLUMN, 5);
249      fail("Should have thrown IllegalArgumentException");
250    } catch (IllegalArgumentException iax) {
251      // success
252    }
253  }
254
255  @Test
256  public void testIncrementOutOfOrder() throws Exception {
257    LOG.info("Starting " + this.name.getMethodName());
258    final TableName TABLENAME =
259        TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
260    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
261
262    byte [][] QUALIFIERS = new byte [][] {
263      Bytes.toBytes("B"), Bytes.toBytes("A"), Bytes.toBytes("C")
264    };
265
266    Increment inc = new Increment(ROW);
267    for (int i=0; i<QUALIFIERS.length; i++) {
268      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
269    }
270    ht.increment(inc);
271
272    // Verify expected results
273    Get get = new Get(ROW);
274    Result r = ht.get(get);
275    Cell [] kvs = r.rawCells();
276    assertEquals(3, kvs.length);
277    assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 1);
278    assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 1);
279    assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1);
280
281    // Now try multiple columns again
282    inc = new Increment(ROW);
283    for (int i=0; i<QUALIFIERS.length; i++) {
284      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
285    }
286    ht.increment(inc);
287
288    // Verify
289    r = ht.get(get);
290    kvs = r.rawCells();
291    assertEquals(3, kvs.length);
292    assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 2);
293    assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 2);
294    assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2);
295  }
296
297  @Test
298  public void testIncrementOnSameColumn() throws Exception {
299    LOG.info("Starting " + this.name.getMethodName());
300    final byte[] TABLENAME = Bytes.toBytes(filterStringSoTableNameSafe(this.name.getMethodName()));
301    Table ht = TEST_UTIL.createTable(TableName.valueOf(TABLENAME), FAMILY);
302
303    byte[][] QUALIFIERS =
304        new byte[][] { Bytes.toBytes("A"), Bytes.toBytes("B"), Bytes.toBytes("C") };
305
306    Increment inc = new Increment(ROW);
307    for (int i = 0; i < QUALIFIERS.length; i++) {
308      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
309      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
310    }
311    ht.increment(inc);
312
313    // Verify expected results
314    Get get = new Get(ROW);
315    Result r = ht.get(get);
316    Cell[] kvs = r.rawCells();
317    assertEquals(3, kvs.length);
318    assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1);
319    assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 1);
320    assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1);
321
322    // Now try multiple columns again
323    inc = new Increment(ROW);
324    for (int i = 0; i < QUALIFIERS.length; i++) {
325      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
326      inc.addColumn(FAMILY, QUALIFIERS[i], 1);
327    }
328    ht.increment(inc);
329
330    // Verify
331    r = ht.get(get);
332    kvs = r.rawCells();
333    assertEquals(3, kvs.length);
334    assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 2);
335    assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 2);
336    assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2);
337
338    ht.close();
339  }
340
341  @Test
342  public void testIncrementIncrZeroAtFirst() throws Exception {
343    LOG.info("Starting " + this.name.getMethodName());
344    final TableName TABLENAME =
345            TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
346    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
347
348    byte[] col1 = Bytes.toBytes("col1");
349    byte[] col2 = Bytes.toBytes("col2");
350    byte[] col3 = Bytes.toBytes("col3");
351
352    // Now increment zero at first time incr
353    Increment inc = new Increment(ROW);
354    inc.addColumn(FAMILY, col1, 0);
355    ht.increment(inc);
356
357    // Verify expected results
358    Get get = new Get(ROW);
359    Result r = ht.get(get);
360    Cell [] kvs = r.rawCells();
361    assertEquals(1, kvs.length);
362    assertNotNull(kvs[0]);
363    assertIncrementKey(kvs[0], ROW, FAMILY, col1, 0);
364
365    // Now try multiple columns by different amounts
366    inc = new Increment(ROW);
367    inc.addColumn(FAMILY, col1, 1);
368    inc.addColumn(FAMILY, col2, 0);
369    inc.addColumn(FAMILY, col3, 2);
370    ht.increment(inc);
371    // Verify
372    get = new Get(ROW);
373    r = ht.get(get);
374    kvs = r.rawCells();
375    assertEquals(3, kvs.length);
376    assertNotNull(kvs[0]);
377    assertNotNull(kvs[1]);
378    assertNotNull(kvs[2]);
379    assertIncrementKey(kvs[0], ROW, FAMILY, col1, 1);
380    assertIncrementKey(kvs[1], ROW, FAMILY, col2, 0);
381    assertIncrementKey(kvs[2], ROW, FAMILY, col3, 2);
382  }
383
384  @Test
385  public void testIncrement() throws Exception {
386    LOG.info("Starting " + this.name.getMethodName());
387    final TableName TABLENAME =
388        TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
389    Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
390
391    byte [][] ROWS = new byte [][] {
392        Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
393        Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
394        Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
395    };
396    byte [][] QUALIFIERS = new byte [][] {
397        Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
398        Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
399        Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
400    };
401
402    // Do some simple single-column increments
403
404    // First with old API
405    ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[0], 1);
406    ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[1], 2);
407    ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[2], 3);
408    ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[3], 4);
409
410    // Now increment things incremented with old and do some new
411    Increment inc = new Increment(ROW);
412    inc.addColumn(FAMILY, QUALIFIERS[1], 1);
413    inc.addColumn(FAMILY, QUALIFIERS[3], 1);
414    inc.addColumn(FAMILY, QUALIFIERS[4], 1);
415    ht.increment(inc);
416
417    // Verify expected results
418    Get get = new Get(ROW);
419    Result r = ht.get(get);
420    Cell [] kvs = r.rawCells();
421    assertEquals(5, kvs.length);
422    assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1);
423    assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 3);
424    assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 3);
425    assertIncrementKey(kvs[3], ROW, FAMILY, QUALIFIERS[3], 5);
426    assertIncrementKey(kvs[4], ROW, FAMILY, QUALIFIERS[4], 1);
427
428    // Now try multiple columns by different amounts
429    inc = new Increment(ROWS[0]);
430    for (int i=0;i<QUALIFIERS.length;i++) {
431      inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
432    }
433    ht.increment(inc);
434    // Verify
435    get = new Get(ROWS[0]);
436    r = ht.get(get);
437    kvs = r.rawCells();
438    assertEquals(QUALIFIERS.length, kvs.length);
439    for (int i=0;i<QUALIFIERS.length;i++) {
440      assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], i+1);
441    }
442
443    // Re-increment them
444    inc = new Increment(ROWS[0]);
445    for (int i=0;i<QUALIFIERS.length;i++) {
446      inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
447    }
448    ht.increment(inc);
449    // Verify
450    r = ht.get(get);
451    kvs = r.rawCells();
452    assertEquals(QUALIFIERS.length, kvs.length);
453    for (int i=0;i<QUALIFIERS.length;i++) {
454      assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1));
455    }
456
457    // Verify that an Increment of an amount of zero, returns current count; i.e. same as for above
458    // test, that is: 2 * (i + 1).
459    inc = new Increment(ROWS[0]);
460    for (int i = 0; i < QUALIFIERS.length; i++) {
461      inc.addColumn(FAMILY, QUALIFIERS[i], 0);
462    }
463    ht.increment(inc);
464    r = ht.get(get);
465    kvs = r.rawCells();
466    assertEquals(QUALIFIERS.length, kvs.length);
467    for (int i = 0; i < QUALIFIERS.length; i++) {
468      assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1));
469    }
470  }
471
472  @Test
473  public void testIncrementWithCustomTimestamp() throws IOException {
474    TableName TABLENAME = TableName.valueOf(name.getMethodName());
475    Table table = TEST_UTIL.createTable(TABLENAME, FAMILY);
476    long timestamp = 999;
477    Increment increment = new Increment(ROW);
478    increment.add(CellUtil.createCell(ROW, FAMILY, QUALIFIER, timestamp, KeyValue.Type.Put.getCode(), Bytes.toBytes(100L)));
479    Result r = table.increment(increment);
480    assertEquals(1, r.size());
481    assertEquals(timestamp, r.rawCells()[0].getTimestamp());
482    r = table.get(new Get(ROW));
483    assertEquals(1, r.size());
484    assertEquals(timestamp, r.rawCells()[0].getTimestamp());
485    r = table.increment(increment);
486    assertEquals(1, r.size());
487    assertNotEquals(timestamp, r.rawCells()[0].getTimestamp());
488    r = table.get(new Get(ROW));
489    assertEquals(1, r.size());
490    assertNotEquals(timestamp, r.rawCells()[0].getTimestamp());
491  }
492
493  /**
494   * Call over to the adjacent class's method of same name.
495   */
496  static void assertIncrementKey(Cell key, byte [] row, byte [] family,
497      byte [] qualifier, long value) throws Exception {
498    TestFromClientSide.assertIncrementKey(key, row, family, qualifier, value);
499  }
500
501  public static String filterStringSoTableNameSafe(final String str) {
502    return str.replaceAll("\\[fast\\=(.*)\\]", ".FAST.is.$1");
503  }
504}