001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.hamcrest.MatcherAssert.assertThat;
021import static org.hamcrest.Matchers.greaterThan;
022import static org.hamcrest.Matchers.lessThan;
023import static org.junit.Assert.assertArrayEquals;
024import static org.junit.Assert.assertEquals;
025import static org.junit.Assert.assertFalse;
026import static org.junit.Assert.assertNull;
027import static org.junit.Assert.assertTrue;
028import static org.junit.Assert.fail;
029
030import java.io.IOException;
031import java.nio.ByteBuffer;
032import java.util.ArrayList;
033import java.util.Arrays;
034import java.util.List;
035import org.apache.hadoop.hbase.ArrayBackedTag;
036import org.apache.hadoop.hbase.ByteBufferKeyValue;
037import org.apache.hadoop.hbase.Cell;
038import org.apache.hadoop.hbase.CellComparator;
039import org.apache.hadoop.hbase.CellScanner;
040import org.apache.hadoop.hbase.CellUtil;
041import org.apache.hadoop.hbase.HBaseClassTestRule;
042import org.apache.hadoop.hbase.KeyValue;
043import org.apache.hadoop.hbase.Tag;
044import org.apache.hadoop.hbase.testclassification.ClientTests;
045import org.apache.hadoop.hbase.testclassification.SmallTests;
046import org.apache.hadoop.hbase.util.ByteBufferUtils;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.junit.ClassRule;
049import org.junit.Test;
050import org.junit.experimental.categories.Category;
051import org.slf4j.Logger;
052import org.slf4j.LoggerFactory;
053
054@Category({ SmallTests.class, ClientTests.class })
055public class TestResult {
056
057  @ClassRule
058  public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestResult.class);
059
060  private static final Logger LOG = LoggerFactory.getLogger(TestResult.class.getName());
061
062  static KeyValue[] genKVs(final byte[] row, final byte[] family, final byte[] value,
063    final long timestamp, final int cols) {
064    KeyValue[] kvs = new KeyValue[cols];
065
066    for (int i = 0; i < cols; i++) {
067      kvs[i] =
068        new KeyValue(row, family, Bytes.toBytes(i), timestamp, Bytes.add(value, Bytes.toBytes(i)));
069    }
070    return kvs;
071  }
072
073  static final byte[] row = Bytes.toBytes("row");
074  static final byte[] family = Bytes.toBytes("family");
075  static final byte[] value = Bytes.toBytes("value");
076  static final byte[] qual = Bytes.toBytes("qual");
077
078  /**
079   * Run some tests to ensure Result acts like a proper CellScanner.
080   */
081  @Test
082  public void testResultAsCellScanner() throws IOException {
083    Cell[] cells = genKVs(row, family, value, 1, 10);
084    Arrays.sort(cells, CellComparator.getInstance());
085    Result r = Result.create(cells);
086    assertSame(r, cells);
087    // Assert I run over same result multiple times.
088    assertSame(r.cellScanner(), cells);
089    assertSame(r.cellScanner(), cells);
090    // Assert we are not creating new object when doing cellscanner
091    assertTrue(r == r.cellScanner());
092  }
093
094  private void assertSame(final CellScanner cellScanner, final Cell[] cells) throws IOException {
095    int count = 0;
096    while (cellScanner.advance()) {
097      assertTrue(cells[count].equals(cellScanner.current()));
098      count++;
099    }
100    assertEquals(cells.length, count);
101  }
102
103  @Test
104  public void testBasicGetColumn() throws Exception {
105    KeyValue[] kvs = genKVs(row, family, value, 1, 100);
106
107    Arrays.sort(kvs, CellComparator.getInstance());
108
109    Result r = Result.create(kvs);
110
111    for (int i = 0; i < 100; ++i) {
112      final byte[] qf = Bytes.toBytes(i);
113
114      List<Cell> ks = r.getColumnCells(family, qf);
115      assertEquals(1, ks.size());
116      assertTrue(CellUtil.matchingQualifier(ks.get(0), qf));
117      assertEquals(ks.get(0), r.getColumnLatestCell(family, qf));
118    }
119  }
120
121  @Test
122  public void testCurrentOnEmptyCell() throws IOException {
123    Result r = Result.create(new Cell[0]);
124    assertFalse(r.advance());
125    assertNull(r.current());
126  }
127
128  @Test
129  public void testAdvanceMultipleOnEmptyCell() throws IOException {
130    Result r = Result.create(new Cell[0]);
131    // After HBASE-26688, advance of result with empty cell list will always return false.
132    // Here 10 is an arbitrary number to test the logic.
133    for (int i = 0; i < 10; i++) {
134      assertFalse(r.advance());
135    }
136  }
137
138  @Test
139  public void testMultiVersionGetColumn() throws Exception {
140    KeyValue[] kvs1 = genKVs(row, family, value, 1, 100);
141    KeyValue[] kvs2 = genKVs(row, family, value, 200, 100);
142
143    KeyValue[] kvs = new KeyValue[kvs1.length + kvs2.length];
144    System.arraycopy(kvs1, 0, kvs, 0, kvs1.length);
145    System.arraycopy(kvs2, 0, kvs, kvs1.length, kvs2.length);
146
147    Arrays.sort(kvs, CellComparator.getInstance());
148
149    Result r = Result.create(kvs);
150    for (int i = 0; i < 100; ++i) {
151      final byte[] qf = Bytes.toBytes(i);
152
153      List<Cell> ks = r.getColumnCells(family, qf);
154      assertEquals(2, ks.size());
155      assertTrue(CellUtil.matchingQualifier(ks.get(0), qf));
156      assertEquals(200, ks.get(0).getTimestamp());
157      assertEquals(ks.get(0), r.getColumnLatestCell(family, qf));
158    }
159  }
160
161  @Test
162  public void testBasicGetValue() throws Exception {
163    KeyValue[] kvs = genKVs(row, family, value, 1, 100);
164
165    Arrays.sort(kvs, CellComparator.getInstance());
166
167    Result r = Result.create(kvs);
168
169    for (int i = 0; i < 100; ++i) {
170      final byte[] qf = Bytes.toBytes(i);
171
172      assertArrayEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf));
173      assertTrue(r.containsColumn(family, qf));
174    }
175  }
176
177  @Test
178  public void testMultiVersionGetValue() throws Exception {
179    KeyValue[] kvs1 = genKVs(row, family, value, 1, 100);
180    KeyValue[] kvs2 = genKVs(row, family, value, 200, 100);
181
182    KeyValue[] kvs = new KeyValue[kvs1.length + kvs2.length];
183    System.arraycopy(kvs1, 0, kvs, 0, kvs1.length);
184    System.arraycopy(kvs2, 0, kvs, kvs1.length, kvs2.length);
185
186    Arrays.sort(kvs, CellComparator.getInstance());
187
188    Result r = Result.create(kvs);
189    for (int i = 0; i < 100; ++i) {
190      final byte[] qf = Bytes.toBytes(i);
191
192      assertArrayEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf));
193      assertTrue(r.containsColumn(family, qf));
194    }
195  }
196
197  @Test
198  public void testBasicLoadValue() throws Exception {
199    KeyValue[] kvs = genKVs(row, family, value, 1, 100);
200
201    Arrays.sort(kvs, CellComparator.getInstance());
202
203    Result r = Result.create(kvs);
204    ByteBuffer loadValueBuffer = ByteBuffer.allocate(1024);
205
206    for (int i = 0; i < 100; ++i) {
207      final byte[] qf = Bytes.toBytes(i);
208
209      loadValueBuffer.clear();
210      r.loadValue(family, qf, loadValueBuffer);
211      loadValueBuffer.flip();
212      assertEquals(loadValueBuffer, ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))));
213      assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))),
214        r.getValueAsByteBuffer(family, qf));
215    }
216  }
217
218  @Test
219  public void testMultiVersionLoadValue() throws Exception {
220    KeyValue[] kvs1 = genKVs(row, family, value, 1, 100);
221    KeyValue[] kvs2 = genKVs(row, family, value, 200, 100);
222
223    KeyValue[] kvs = new KeyValue[kvs1.length + kvs2.length];
224    System.arraycopy(kvs1, 0, kvs, 0, kvs1.length);
225    System.arraycopy(kvs2, 0, kvs, kvs1.length, kvs2.length);
226
227    Arrays.sort(kvs, CellComparator.getInstance());
228
229    ByteBuffer loadValueBuffer = ByteBuffer.allocate(1024);
230
231    Result r = Result.create(kvs);
232    for (int i = 0; i < 100; ++i) {
233      final byte[] qf = Bytes.toBytes(i);
234
235      loadValueBuffer.clear();
236      r.loadValue(family, qf, loadValueBuffer);
237      loadValueBuffer.flip();
238      assertEquals(loadValueBuffer, ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))));
239      assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))),
240        r.getValueAsByteBuffer(family, qf));
241    }
242  }
243
244  /**
245   * Verify that Result.compareResults(...) behaves correctly.
246   */
247  @Test
248  public void testCompareResults() throws Exception {
249    byte[] value1 = Bytes.toBytes("value1");
250    byte[] qual = Bytes.toBytes("qual");
251
252    KeyValue kv1 = new KeyValue(row, family, qual, value);
253    KeyValue kv2 = new KeyValue(row, family, qual, value1);
254
255    Result r1 = Result.create(new KeyValue[] { kv1 });
256    Result r2 = Result.create(new KeyValue[] { kv2 });
257    // no exception thrown
258    Result.compareResults(r1, r1);
259    try {
260      // these are different (HBASE-4800)
261      Result.compareResults(r1, r2);
262      fail();
263    } catch (Exception x) {
264      assertTrue(x.getMessage().startsWith("This result was different:"));
265    }
266  }
267
268  @Test
269  public void testCompareResultsWithTags() throws Exception {
270    Tag t1 = new ArrayBackedTag((byte) 1, Bytes.toBytes("TAG1"));
271    Tag t2 = new ArrayBackedTag((byte) 2, Bytes.toBytes("TAG2"));
272    // Both BB backed tags KV are null
273    Result result1 = getByteBufferBackedTagResult(null);
274    Result result2 = getByteBufferBackedTagResult(null);
275    Result.compareResults(result1, result2);
276
277    // Test both byte buffer backed tags KeyValue
278    result1 = getByteBufferBackedTagResult(t1);
279    result2 = getByteBufferBackedTagResult(t1);
280    Result.compareResults(result1, result2);
281
282    // Both array backed tags KV are null
283    result1 = getArrayBackedTagResult(null);
284    result2 = getArrayBackedTagResult(null);
285    Result.compareResults(result1, result2);
286
287    // Test both array backed tags KeyValue
288    result1 = getArrayBackedTagResult(t1);
289    result2 = getArrayBackedTagResult(t1);
290    Result.compareResults(result1, result2);
291
292    // left instance of byte buffer and right instance of array backed
293    result1 = getByteBufferBackedTagResult(t1);
294    result2 = getArrayBackedTagResult(t1);
295    Result.compareResults(result1, result2);
296
297    // left instance of array backed and right instance of byte buffer backed.
298    result1 = getArrayBackedTagResult(t1);
299    result2 = getByteBufferBackedTagResult(t1);
300    Result.compareResults(result1, result2);
301
302    // Left BB backed null tag and right BB backed non null tag
303    result1 = getByteBufferBackedTagResult(null);
304    result2 = getByteBufferBackedTagResult(t2);
305    try {
306      Result.compareResults(result1, result2);
307      fail();
308    } catch (Exception e) {
309      // Expected
310    }
311
312    // Left BB backed non null tag and right BB backed null tag
313    result1 = getByteBufferBackedTagResult(t1);
314    result2 = getByteBufferBackedTagResult(null);
315    try {
316      Result.compareResults(result1, result2);
317      fail();
318    } catch (Exception e) {
319      // Expected
320    }
321
322    // Both byte buffer backed tags KV are different
323    result1 = getByteBufferBackedTagResult(t1);
324    result2 = getByteBufferBackedTagResult(t2);
325    try {
326      Result.compareResults(result1, result2);
327      fail();
328    } catch (Exception e) {
329      // Expected
330    }
331
332    // Left array backed non null tag and right array backed null tag
333    result1 = getArrayBackedTagResult(t1);
334    result2 = getArrayBackedTagResult(null);
335    try {
336      Result.compareResults(result1, result2);
337      fail();
338    } catch (Exception e) {
339      // Expected
340    }
341
342    // Left array backed null tag and right array backed non null tag
343    result1 = getByteBufferBackedTagResult(null);
344    result2 = getByteBufferBackedTagResult(t2);
345    try {
346      Result.compareResults(result1, result2);
347      fail();
348    } catch (Exception e) {
349      // Expected
350    }
351
352    // Both array backed tags KV are different
353    result1 = getArrayBackedTagResult(t1);
354    result2 = getArrayBackedTagResult(t2);
355    try {
356      Result.compareResults(result1, result2);
357      fail();
358    } catch (Exception e) {
359      // Expected
360    }
361
362    // left instance of byte buffer and right instance of array backed are different
363    result1 = getByteBufferBackedTagResult(t1);
364    result2 = getArrayBackedTagResult(t2);
365    try {
366      Result.compareResults(result1, result2);
367      fail();
368    } catch (Exception e) {
369      // Expected
370    }
371
372    // left instance of array backed and right instance of byte buffer backed are different
373    result1 = getArrayBackedTagResult(t1);
374    result2 = getByteBufferBackedTagResult(t2);
375    try {
376      Result.compareResults(result1, result2);
377      fail();
378    } catch (Exception e) {
379      // Expected
380    }
381  }
382
383  @Test
384  public void testCompareResultMemoryUsage() {
385    List<Cell> cells1 = new ArrayList<>();
386    for (long i = 0; i < 100; i++) {
387      cells1.add(new KeyValue(row, family, Bytes.toBytes(i), value));
388    }
389
390    List<Cell> cells2 = new ArrayList<>();
391    for (long i = 0; i < 100; i++) {
392      cells2.add(new KeyValue(row, family, Bytes.toBytes(i), Bytes.toBytes(i)));
393    }
394
395    Result r1 = Result.create(cells1);
396    Result r2 = Result.create(cells2);
397    try {
398      Result.compareResults(r1, r2);
399      fail();
400    } catch (Exception x) {
401      assertTrue(x.getMessage().startsWith("This result was different:"));
402      assertThat(x.getMessage().length(), greaterThan(100));
403    }
404
405    try {
406      Result.compareResults(r1, r2, false);
407      fail();
408    } catch (Exception x) {
409      assertEquals("This result was different: row=row", x.getMessage());
410      assertThat(x.getMessage().length(), lessThan(100));
411    }
412  }
413
414  private Result getArrayBackedTagResult(Tag tag) {
415    List<Tag> tags = null;
416    if (tag != null) {
417      tags = Arrays.asList(tag);
418    }
419    KeyValue kvCell = new KeyValue(row, family, qual, 0L, KeyValue.Type.Put, value, tags);
420    return Result.create(new Cell[] { kvCell });
421  }
422
423  private Result getByteBufferBackedTagResult(Tag tag) {
424    List<Tag> tags = null;
425    if (tag != null) {
426      tags = Arrays.asList(tag);
427    }
428    KeyValue kvCell = new KeyValue(row, family, qual, 0L, KeyValue.Type.Put, value, tags);
429    ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length);
430    ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length);
431    ByteBufferKeyValue bbKV = new ByteBufferKeyValue(buf, 0, buf.capacity(), 0L);
432    return Result.create(new Cell[] { bbKV });
433  }
434
435  /**
436   * Verifies that one can't modify instance of EMPTY_RESULT.
437   */
438  @Test
439  public void testEmptyResultIsReadonly() {
440    Result emptyResult = Result.EMPTY_RESULT;
441    Result otherResult = new Result();
442
443    try {
444      emptyResult.copyFrom(otherResult);
445      fail("UnsupportedOperationException should have been thrown!");
446    } catch (UnsupportedOperationException ex) {
447      LOG.debug("As expected: " + ex.getMessage());
448    }
449    try {
450      emptyResult.setExists(true);
451      fail("UnsupportedOperationException should have been thrown!");
452    } catch (UnsupportedOperationException ex) {
453      LOG.debug("As expected: " + ex.getMessage());
454    }
455  }
456
457  /**
458   * Microbenchmark that compares {@link Result#getValue} and {@link Result#loadValue} performance.
459   * n
460   */
461  public void doReadBenchmark() throws Exception {
462
463    final int n = 5;
464    final int m = 100000000;
465
466    StringBuilder valueSB = new StringBuilder();
467    for (int i = 0; i < 100; i++) {
468      valueSB.append((byte) (Math.random() * 10));
469    }
470
471    StringBuilder rowSB = new StringBuilder();
472    for (int i = 0; i < 50; i++) {
473      rowSB.append((byte) (Math.random() * 10));
474    }
475
476    KeyValue[] kvs =
477      genKVs(Bytes.toBytes(rowSB.toString()), family, Bytes.toBytes(valueSB.toString()), 1, n);
478    Arrays.sort(kvs, CellComparator.getInstance());
479    ByteBuffer loadValueBuffer = ByteBuffer.allocate(1024);
480    Result r = Result.create(kvs);
481
482    byte[][] qfs = new byte[n][Bytes.SIZEOF_INT];
483    for (int i = 0; i < n; ++i) {
484      System.arraycopy(qfs[i], 0, Bytes.toBytes(i), 0, Bytes.SIZEOF_INT);
485    }
486
487    // warm up
488    for (int k = 0; k < 100000; k++) {
489      for (int i = 0; i < n; ++i) {
490        r.getValue(family, qfs[i]);
491        loadValueBuffer.clear();
492        r.loadValue(family, qfs[i], loadValueBuffer);
493        loadValueBuffer.flip();
494      }
495    }
496
497    System.gc();
498    long start = System.nanoTime();
499    for (int k = 0; k < m; k++) {
500      for (int i = 0; i < n; ++i) {
501        loadValueBuffer.clear();
502        r.loadValue(family, qfs[i], loadValueBuffer);
503        loadValueBuffer.flip();
504      }
505    }
506    long stop = System.nanoTime();
507    System.out.println("loadValue(): " + (stop - start));
508
509    System.gc();
510    start = System.nanoTime();
511    for (int k = 0; k < m; k++) {
512      for (int i = 0; i < n; i++) {
513        r.getValue(family, qfs[i]);
514      }
515    }
516    stop = System.nanoTime();
517    System.out.println("getValue():  " + (stop - start));
518  }
519
520  /**
521   * Calls non-functional test methods. n
522   */
523  public static void main(String[] args) {
524    TestResult testResult = new TestResult();
525    try {
526      testResult.doReadBenchmark();
527    } catch (Exception e) {
528      LOG.error("Unexpected exception", e);
529    }
530  }
531}