001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.Assert.assertArrayEquals;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertNotEquals;
024import static org.junit.Assert.assertNotNull;
025import static org.junit.Assert.assertTrue;
026import static org.junit.Assert.fail;
027
028import java.io.ByteArrayInputStream;
029import java.io.ByteArrayOutputStream;
030import java.io.DataInputStream;
031import java.io.DataOutputStream;
032import java.util.Collections;
033import java.util.Iterator;
034import java.util.List;
035import java.util.Set;
036import java.util.TreeSet;
037import org.apache.hadoop.hbase.testclassification.SmallTests;
038import org.apache.hadoop.hbase.util.ByteBufferUtils;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
041import org.junit.ClassRule;
042import org.junit.Test;
043import org.junit.experimental.categories.Category;
044import org.slf4j.Logger;
045import org.slf4j.LoggerFactory;
046
047@Category(SmallTests.class)
048public class TestKeyValue {
049  @ClassRule
050  public static final HBaseClassTestRule CLASS_RULE =
051    HBaseClassTestRule.forClass(TestKeyValue.class);
052  private static final Logger LOG = LoggerFactory.getLogger(TestKeyValue.class);
053
054  @Test
055  public void testColumnCompare() {
056    final byte[] a = Bytes.toBytes("aaa");
057    byte[] family1 = Bytes.toBytes("abc");
058    byte[] qualifier1 = Bytes.toBytes("def");
059    byte[] family2 = Bytes.toBytes("abcd");
060    byte[] qualifier2 = Bytes.toBytes("ef");
061
062    KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, KeyValue.Type.Put, a);
063    assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
064    assertTrue(CellUtil.matchingColumn(aaa, family1, qualifier1));
065    aaa = new KeyValue(a, family2, qualifier2, 0L, KeyValue.Type.Put, a);
066    assertFalse(CellUtil.matchingColumn(aaa, family1, qualifier1));
067    assertTrue(CellUtil.matchingColumn(aaa, family2, qualifier2));
068    byte[] nullQualifier = new byte[0];
069    aaa = new KeyValue(a, family1, nullQualifier, 0L, KeyValue.Type.Put, a);
070    assertTrue(CellUtil.matchingColumn(aaa, family1, null));
071    assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
072  }
073
074  /**
075   * Test a corner case when the family qualifier is a prefix of the column qualifier.
076   */
077  @Test
078  public void testColumnCompare_prefix() {
079    final byte[] a = Bytes.toBytes("aaa");
080    byte[] family1 = Bytes.toBytes("abc");
081    byte[] qualifier1 = Bytes.toBytes("def");
082    byte[] family2 = Bytes.toBytes("ab");
083    byte[] qualifier2 = Bytes.toBytes("def");
084
085    KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, KeyValue.Type.Put, a);
086    assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
087  }
088
089  @Test
090  public void testBasics() {
091    LOG.info("LOWKEY: " + KeyValue.LOWESTKEY.toString());
092    String name = "testBasics";
093    check(Bytes.toBytes(name), Bytes.toBytes(name), Bytes.toBytes(name), 1, Bytes.toBytes(name));
094    // Test empty value and empty column -- both should work. (not empty fam)
095    check(Bytes.toBytes(name), Bytes.toBytes(name), null, 1, null);
096    check(HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes(name), null, 1, null);
097    // empty qual is equivalent to null qual
098    assertEquals(new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"), null, 1, (byte[]) null),
099      new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"), HConstants.EMPTY_BYTE_ARRAY, 1,
100        (byte[]) null));
101  }
102
103  private void check(final byte[] row, final byte[] family, byte[] qualifier, final long timestamp,
104    final byte[] value) {
105    KeyValue kv = new KeyValue(row, family, qualifier, timestamp, value);
106    assertTrue(
107      Bytes.compareTo(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), row, 0, row.length)
108          == 0);
109    assertTrue(CellUtil.matchingColumn(kv, family, qualifier));
110    // Call toString to make sure it works.
111    LOG.info(kv.toString());
112  }
113
114  @Test
115  public void testPlainCompare() {
116    final byte[] a = Bytes.toBytes("aaa");
117    final byte[] b = Bytes.toBytes("bbb");
118    final byte[] fam = Bytes.toBytes("col");
119    final byte[] qf = Bytes.toBytes("umn");
120    KeyValue aaa = new KeyValue(a, fam, qf, a);
121    KeyValue bbb = new KeyValue(b, fam, qf, b);
122    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) < 0);
123    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) > 0);
124    // Compare breaks if passed same ByteBuffer as both left and right arguments.
125    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, bbb) == 0);
126    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0);
127    // Do compare with different timestamps.
128    aaa = new KeyValue(a, fam, qf, 1, a);
129    bbb = new KeyValue(a, fam, qf, 2, a);
130    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) > 0);
131    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) < 0);
132    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0);
133    // Do compare with different types. Higher numbered types -- Delete
134    // should sort ahead of lower numbers; i.e. Put
135    aaa = new KeyValue(a, fam, qf, 1, KeyValue.Type.Delete, a);
136    bbb = new KeyValue(a, fam, qf, 1, a);
137    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) < 0);
138    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) > 0);
139    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0);
140  }
141
142  @Test
143  public void testMoreComparisons() {
144    long now = EnvironmentEdgeManager.currentTime();
145
146    // Meta compares
147    KeyValue aaa =
148      new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236020145502"), now);
149    KeyValue bbb = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,,99999999999999"), now);
150    CellComparator c = MetaCellComparator.META_COMPARATOR;
151    assertTrue(c.compare(bbb, aaa) < 0);
152
153    KeyValue aaaa = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,,1236023996656"),
154      Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236024396271L, (byte[]) null);
155    assertTrue(c.compare(aaaa, bbb) < 0);
156
157    KeyValue x = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
158      Bytes.toBytes("info"), Bytes.toBytes(""), 9223372036854775807L, (byte[]) null);
159    KeyValue y = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
160      Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236034574912L, (byte[]) null);
161    assertTrue(c.compare(x, y) < 0);
162    comparisons(MetaCellComparator.META_COMPARATOR);
163    comparisons(CellComparatorImpl.COMPARATOR);
164    metacomparisons(MetaCellComparator.META_COMPARATOR);
165  }
166
167  @Test
168  public void testMetaComparatorTableKeysWithCommaOk() {
169    CellComparator c = MetaCellComparator.META_COMPARATOR;
170    long now = EnvironmentEdgeManager.currentTime();
171    // meta keys values are not quite right. A users can enter illegal values
172    // from shell when scanning meta.
173    KeyValue a = new KeyValue(Bytes.toBytes("table,key,with,commas1,1234"), now);
174    KeyValue b = new KeyValue(Bytes.toBytes("table,key,with,commas2,0123"), now);
175    assertTrue(c.compare(a, b) < 0);
176  }
177
178  /**
179   * Tests cases where rows keys have characters below the ','. See HBASE-832
180   */
181  @Test
182  public void testKeyValueBorderCases() {
183    // % sorts before , so if we don't do special comparator, rowB would
184    // come before rowA.
185    KeyValue rowA = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/,1234"),
186      Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[]) null);
187    KeyValue rowB = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/%20,99999"),
188      Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[]) null);
189    assertTrue(MetaCellComparator.META_COMPARATOR.compare(rowA, rowB) < 0);
190
191    rowA = new KeyValue(Bytes.toBytes("testtable,,1234"), Bytes.toBytes("fam"), Bytes.toBytes(""),
192      Long.MAX_VALUE, (byte[]) null);
193    rowB = new KeyValue(Bytes.toBytes("testtable,$www.hbase.org/,99999"), Bytes.toBytes("fam"),
194      Bytes.toBytes(""), Long.MAX_VALUE, (byte[]) null);
195    assertTrue(MetaCellComparator.META_COMPARATOR.compare(rowA, rowB) < 0);
196  }
197
198  private void metacomparisons(final CellComparatorImpl c) {
199    long now = EnvironmentEdgeManager.currentTime();
200    assertTrue(c.compare(
201      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now),
202      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now))
203        == 0);
204    KeyValue a =
205      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now);
206    KeyValue b =
207      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,2"), now);
208    assertTrue(c.compare(a, b) < 0);
209    assertTrue(c.compare(
210      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,2"), now),
211      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now))
212        > 0);
213  }
214
215  private void comparisons(final CellComparatorImpl c) {
216    long now = EnvironmentEdgeManager.currentTime();
217    assertTrue(c.compare(
218      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now),
219      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now)) == 0);
220    assertTrue(c.compare(
221      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now),
222      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,2"), now)) < 0);
223    assertTrue(c.compare(
224      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,2"), now),
225      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now)) > 0);
226  }
227
228  @Test
229  public void testBinaryKeys() {
230    Set<KeyValue> set = new TreeSet<>(CellComparatorImpl.COMPARATOR);
231    final byte[] fam = Bytes.toBytes("col");
232    final byte[] qf = Bytes.toBytes("umn");
233    final byte[] nb = new byte[0];
234    KeyValue[] keys = { new KeyValue(Bytes.toBytes("aaaaa,\u0000\u0000,2"), fam, qf, 2, nb),
235      new KeyValue(Bytes.toBytes("aaaaa,\u0001,3"), fam, qf, 3, nb),
236      new KeyValue(Bytes.toBytes("aaaaa,,1"), fam, qf, 1, nb),
237      new KeyValue(Bytes.toBytes("aaaaa,\u1000,5"), fam, qf, 5, nb),
238      new KeyValue(Bytes.toBytes("aaaaa,a,4"), fam, qf, 4, nb),
239      new KeyValue(Bytes.toBytes("a,a,0"), fam, qf, 0, nb), };
240    // Add to set with bad comparator
241    Collections.addAll(set, keys);
242    // This will output the keys incorrectly.
243    boolean assertion = false;
244    int count = 0;
245    try {
246      for (KeyValue k : set) {
247        assertEquals(count++, k.getTimestamp());
248      }
249    } catch (java.lang.AssertionError e) {
250      // Expected
251      assertion = true;
252    }
253    assertTrue(assertion);
254    // Make set with good comparator
255    set = new TreeSet<>(MetaCellComparator.META_COMPARATOR);
256    Collections.addAll(set, keys);
257    count = 0;
258    for (KeyValue k : set) {
259      assertEquals(count++, k.getTimestamp());
260    }
261  }
262
263  @Test
264  public void testStackedUpKeyValue() {
265    // Test multiple KeyValues in a single blob.
266
267    // TODO actually write this test!
268  }
269
270  private final byte[] rowA = Bytes.toBytes("rowA");
271  private final byte[] rowB = Bytes.toBytes("rowB");
272
273  private final byte[] family = Bytes.toBytes("family");
274  private final byte[] qualA = Bytes.toBytes("qfA");
275
276  private void assertKVLess(CellComparator c, KeyValue less, KeyValue greater) {
277    int cmp = c.compare(less, greater);
278    assertTrue(cmp < 0);
279    cmp = c.compare(greater, less);
280    assertTrue(cmp > 0);
281  }
282
283  private void assertKVLessWithoutRow(CellComparator c, KeyValue less, KeyValue greater) {
284    int cmp = c.compare(less, greater);
285    assertTrue(cmp < 0);
286    cmp = c.compare(greater, less);
287    assertTrue(cmp > 0);
288  }
289
290  @Test
291  public void testCompareWithoutRow() {
292    final CellComparator c = CellComparatorImpl.COMPARATOR;
293    byte[] row = Bytes.toBytes("row");
294
295    byte[] fa = Bytes.toBytes("fa");
296    byte[] fami = Bytes.toBytes("fami");
297    byte[] fami1 = Bytes.toBytes("fami1");
298
299    byte[] qual0 = Bytes.toBytes("");
300    byte[] qual1 = Bytes.toBytes("qf1");
301    byte[] qual2 = Bytes.toBytes("qf2");
302    long ts = 1;
303
304    // 'fa:'
305    KeyValue kv_0 = new KeyValue(row, fa, qual0, ts, KeyValue.Type.Put);
306    // 'fami:'
307    KeyValue kv0_0 = new KeyValue(row, fami, qual0, ts, KeyValue.Type.Put);
308    // 'fami:qf1'
309    KeyValue kv0_1 = new KeyValue(row, fami, qual1, ts, KeyValue.Type.Put);
310    // 'fami:qf2'
311    KeyValue kv0_2 = new KeyValue(row, fami, qual2, ts, KeyValue.Type.Put);
312    // 'fami1:'
313    KeyValue kv1_0 = new KeyValue(row, fami1, qual0, ts, KeyValue.Type.Put);
314
315    // 'fami:qf1' < 'fami:qf2'
316    assertKVLessWithoutRow(c, kv0_1, kv0_2);
317    // 'fami:qf1' < 'fami1:'
318    assertKVLessWithoutRow(c, kv0_1, kv1_0);
319
320    // Test comparison by skipping the same prefix bytes.
321    /*
322     * KeyValue Format and commonLength:
323     * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|....
324     * ------------------|-------commonLength--------|--------------
325     */
326    int commonLength = KeyValue.ROW_LENGTH_SIZE + KeyValue.FAMILY_LENGTH_SIZE + row.length;
327    // 'fa:' < 'fami:'. They have commonPrefix + 2 same prefix bytes.
328    assertKVLessWithoutRow(c, kv_0, kv0_0);
329    // 'fami:' < 'fami:qf1'. They have commonPrefix + 4 same prefix bytes.
330    assertKVLessWithoutRow(c, kv0_0, kv0_1);
331    // 'fami:qf1' < 'fami1:'. They have commonPrefix + 4 same prefix bytes.
332    assertKVLessWithoutRow(c, kv0_1, kv1_0);
333    // 'fami:qf1' < 'fami:qf2'. They have commonPrefix + 6 same prefix bytes.
334    assertKVLessWithoutRow(c, kv0_1, kv0_2);
335  }
336
337  @Test
338  public void testFirstLastOnRow() {
339    final CellComparator c = CellComparatorImpl.COMPARATOR;
340    long ts = 1;
341    byte[] bufferA = new byte[128];
342    int offsetA = 0;
343    byte[] bufferB = new byte[128];
344    int offsetB = 7;
345
346    // These are listed in sort order (ie: every one should be less
347    // than the one on the next line).
348    final KeyValue firstOnRowA = KeyValueUtil.createFirstOnRow(rowA);
349    final KeyValue firstOnRowABufferFamQual = KeyValueUtil.createFirstOnRow(bufferA, offsetA, rowA,
350      0, rowA.length, family, 0, family.length, qualA, 0, qualA.length);
351    final KeyValue kvA_1 = new KeyValue(rowA, null, null, ts, KeyValue.Type.Put);
352    final KeyValue kvA_2 = new KeyValue(rowA, family, qualA, ts, KeyValue.Type.Put);
353
354    final KeyValue lastOnRowA = KeyValueUtil.createLastOnRow(rowA);
355    final KeyValue firstOnRowB = KeyValueUtil.createFirstOnRow(rowB);
356    final KeyValue firstOnRowBBufferFam = KeyValueUtil.createFirstOnRow(bufferB, offsetB, rowB, 0,
357      rowB.length, family, 0, family.length, null, 0, 0);
358    final KeyValue kvB = new KeyValue(rowB, family, qualA, ts, KeyValue.Type.Put);
359
360    assertKVLess(c, firstOnRowA, firstOnRowB);
361    assertKVLess(c, firstOnRowA, firstOnRowBBufferFam);
362    assertKVLess(c, firstOnRowABufferFamQual, firstOnRowB);
363    assertKVLess(c, firstOnRowA, kvA_1);
364    assertKVLess(c, firstOnRowA, kvA_2);
365    assertKVLess(c, firstOnRowABufferFamQual, kvA_2);
366    assertKVLess(c, kvA_1, kvA_2);
367    assertKVLess(c, kvA_2, firstOnRowB);
368    assertKVLess(c, kvA_1, firstOnRowB);
369    assertKVLess(c, kvA_2, firstOnRowBBufferFam);
370    assertKVLess(c, kvA_1, firstOnRowBBufferFam);
371
372    assertKVLess(c, lastOnRowA, firstOnRowB);
373    assertKVLess(c, lastOnRowA, firstOnRowBBufferFam);
374    assertKVLess(c, firstOnRowB, kvB);
375    assertKVLess(c, firstOnRowBBufferFam, kvB);
376    assertKVLess(c, lastOnRowA, kvB);
377
378    assertKVLess(c, kvA_2, lastOnRowA);
379    assertKVLess(c, kvA_1, lastOnRowA);
380    assertKVLess(c, firstOnRowA, lastOnRowA);
381    assertKVLess(c, firstOnRowABufferFamQual, lastOnRowA);
382  }
383
384  @Test
385  public void testCreateKeyOnly() {
386    long ts = 1;
387    byte[] value = Bytes.toBytes("a real value");
388    byte[] evalue = new byte[0]; // empty value
389
390    for (byte[] val : new byte[][] { value, evalue }) {
391      for (boolean useLen : new boolean[] { false, true }) {
392        KeyValue kv1 = new KeyValue(rowA, family, qualA, ts, val);
393        KeyValue kv1ko = kv1.createKeyOnly(useLen);
394        // keys are still the same
395        assertTrue(kv1.equals(kv1ko));
396        // but values are not
397        assertTrue(kv1ko.getValueLength() == (useLen ? Bytes.SIZEOF_INT : 0));
398        if (useLen) {
399          assertEquals(kv1.getValueLength(),
400            Bytes.toInt(kv1ko.getValueArray(), kv1ko.getValueOffset(), kv1ko.getValueLength()));
401        }
402      }
403    }
404  }
405
406  @Test
407  public void testCreateKeyValueFromKey() {
408    KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
409      Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("myValue"));
410    int initialPadding = 10;
411    int endingPadding = 20;
412    int keyLen = kv.getKeyLength();
413    byte[] tmpArr = new byte[initialPadding + endingPadding + keyLen];
414    System.arraycopy(kv.getBuffer(), kv.getKeyOffset(), tmpArr, initialPadding, keyLen);
415    KeyValue kvFromKey = KeyValueUtil.createKeyValueFromKey(tmpArr, initialPadding, keyLen);
416    assertEquals(keyLen, kvFromKey.getKeyLength());
417    assertEquals(KeyValue.ROW_OFFSET + keyLen, kvFromKey.getBuffer().length);
418    System.err.println("kv=" + kv);
419    System.err.println("kvFromKey=" + kvFromKey);
420    assertEquals(kvFromKey.toString(), kv.toString().replaceAll("=[0-9]+", "=0"));
421  }
422
423  /**
424   * Tests that getTimestamp() does always return the proper timestamp, even after updating it. See
425   * HBASE-6265.
426   */
427  @Test
428  public void testGetTimestamp() {
429    KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
430      Bytes.toBytes("myQualifier"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("myValue"));
431    long time1 = kv.getTimestamp();
432    kv.updateLatestStamp(Bytes.toBytes(12345L));
433    long time2 = kv.getTimestamp();
434    assertEquals(HConstants.LATEST_TIMESTAMP, time1);
435    assertEquals(12345L, time2);
436  }
437
438  @Test
439  public void testKVsWithTags() {
440    byte[] row = Bytes.toBytes("myRow");
441    byte[] cf = Bytes.toBytes("myCF");
442    byte[] q = Bytes.toBytes("myQualifier");
443    byte[] value = Bytes.toBytes("myValue");
444    byte[] metaValue1 = Bytes.toBytes("metaValue1");
445    byte[] metaValue2 = Bytes.toBytes("metaValue2");
446    KeyValue kv = new KeyValue(row, cf, q, HConstants.LATEST_TIMESTAMP, value, new Tag[] {
447      new ArrayBackedTag((byte) 1, metaValue1), new ArrayBackedTag((byte) 2, metaValue2) });
448    assertTrue(kv.getTagsLength() > 0);
449    assertTrue(
450      Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), row, 0, row.length));
451    assertTrue(Bytes.equals(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), cf, 0,
452      cf.length));
453    assertTrue(Bytes.equals(kv.getQualifierArray(), kv.getQualifierOffset(),
454      kv.getQualifierLength(), q, 0, q.length));
455    assertTrue(Bytes.equals(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength(), value, 0,
456      value.length));
457    List<Tag> tags = PrivateCellUtil.getTags(kv);
458    assertNotNull(tags);
459    assertEquals(2, tags.size());
460    boolean meta1Ok = false, meta2Ok = false;
461    for (Tag tag : tags) {
462      if (tag.getType() == (byte) 1) {
463        if (Bytes.equals(Tag.cloneValue(tag), metaValue1)) {
464          meta1Ok = true;
465        }
466      } else {
467        if (Bytes.equals(Tag.cloneValue(tag), metaValue2)) {
468          meta2Ok = true;
469        }
470      }
471    }
472    assertTrue(meta1Ok);
473    assertTrue(meta2Ok);
474    Iterator<Tag> tagItr = PrivateCellUtil.tagsIterator(kv);
475
476    assertTrue(tagItr.hasNext());
477    Tag next = tagItr.next();
478    assertEquals(10, next.getValueLength());
479    assertEquals((byte) 1, next.getType());
480    Bytes.equals(Tag.cloneValue(next), metaValue1);
481    assertTrue(tagItr.hasNext());
482    next = tagItr.next();
483    assertEquals(10, next.getValueLength());
484    assertEquals((byte) 2, next.getType());
485    Bytes.equals(Tag.cloneValue(next), metaValue2);
486    assertFalse(tagItr.hasNext());
487
488    tagItr = PrivateCellUtil.tagsIterator(kv);
489    assertTrue(tagItr.hasNext());
490    next = tagItr.next();
491    assertEquals(10, next.getValueLength());
492    assertEquals((byte) 1, next.getType());
493    Bytes.equals(Tag.cloneValue(next), metaValue1);
494    assertTrue(tagItr.hasNext());
495    next = tagItr.next();
496    assertEquals(10, next.getValueLength());
497    assertEquals((byte) 2, next.getType());
498    Bytes.equals(Tag.cloneValue(next), metaValue2);
499    assertFalse(tagItr.hasNext());
500  }
501
502  @Test
503  public void testMetaKeyComparator() {
504    CellComparator c = MetaCellComparator.META_COMPARATOR;
505    long now = EnvironmentEdgeManager.currentTime();
506
507    KeyValue a = new KeyValue(Bytes.toBytes("table1"), now);
508    KeyValue b = new KeyValue(Bytes.toBytes("table2"), now);
509    assertTrue(c.compare(a, b) < 0);
510
511    a = new KeyValue(Bytes.toBytes("table1,111"), now);
512    b = new KeyValue(Bytes.toBytes("table2"), now);
513    assertTrue(c.compare(a, b) < 0);
514
515    a = new KeyValue(Bytes.toBytes("table1"), now);
516    b = new KeyValue(Bytes.toBytes("table2,111"), now);
517    assertTrue(c.compare(a, b) < 0);
518
519    a = new KeyValue(Bytes.toBytes("table,111"), now);
520    b = new KeyValue(Bytes.toBytes("table,2222"), now);
521    assertTrue(c.compare(a, b) < 0);
522
523    a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
524    b = new KeyValue(Bytes.toBytes("table,2222"), now);
525    assertTrue(c.compare(a, b) < 0);
526
527    a = new KeyValue(Bytes.toBytes("table,111"), now);
528    b = new KeyValue(Bytes.toBytes("table,2222.bbb"), now);
529    assertTrue(c.compare(a, b) < 0);
530
531    a = new KeyValue(Bytes.toBytes("table,,aaaa"), now);
532    b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
533    assertTrue(c.compare(a, b) < 0);
534
535    a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
536    b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
537    assertTrue(c.compare(a, b) < 0);
538
539    a = new KeyValue(Bytes.toBytes("table,111,xxxx"), now);
540    b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
541    assertTrue(c.compare(a, b) < 0);
542
543    a = new KeyValue(Bytes.toBytes("table,111,11,xxx"), now);
544    b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
545    assertTrue(c.compare(a, b) < 0);
546  }
547
548  @Test
549  public void testEqualsAndHashCode() {
550    KeyValue kvA1 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
551      Bytes.toBytes("1"));
552    KeyValue kvA2 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
553      Bytes.toBytes("2"));
554    // We set a different sequence id on kvA2 to demonstrate that the equals and hashCode also
555    // don't take this into account.
556    kvA2.setSequenceId(2);
557    KeyValue kvB = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualB"),
558      Bytes.toBytes("1"));
559
560    assertEquals(kvA1, kvA2);
561    assertNotEquals(kvA1, kvB);
562    assertEquals(kvA1.hashCode(), kvA2.hashCode());
563    assertNotEquals(kvA1.hashCode(), kvB.hashCode());
564  }
565
566  @Test
567  public void testKeyValueSerialization() throws Exception {
568    KeyValue[] keyValues = new KeyValue[] {
569      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
570        Bytes.toBytes("1")),
571      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
572        Bytes.toBytes("2")),
573      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
574        EnvironmentEdgeManager.currentTime(), Bytes.toBytes("2"),
575        new Tag[] { new ArrayBackedTag((byte) 120, "tagA"),
576          new ArrayBackedTag((byte) 121, Bytes.toBytes("tagB")) }),
577      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
578        EnvironmentEdgeManager.currentTime(), Bytes.toBytes("2"),
579        new Tag[] { new ArrayBackedTag((byte) 0, "tagA") }),
580      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes(""),
581        Bytes.toBytes("1")) };
582    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
583    for (KeyValue kv : keyValues) {
584      DataOutputStream os = new DataOutputStream(byteArrayOutputStream);
585      ByteBufferUtils.putInt(os, KeyValueUtil.getSerializedSize(kv, true));
586      KeyValueUtil.oswrite(kv, os, true);
587    }
588    DataInputStream is =
589      new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()));
590    for (int i = 0; i < keyValues.length; i++) {
591      LOG.info("Case#" + i + ": deserialize the kv: " + keyValues[i]);
592      KeyValue destKv = KeyValueUtil.createKeyValueFromInputStream(is, true);
593      assertEquals(keyValues[i], destKv);
594      assertArrayEquals(CellUtil.cloneValue(keyValues[i]), CellUtil.cloneValue(destKv));
595      assertArrayEquals(PrivateCellUtil.cloneTags(keyValues[i]), PrivateCellUtil.cloneTags(destKv));
596    }
597  }
598
599  @Test
600  public void testNullByteArrayKeyValueFailure() {
601    // can't add to testCheckKeyValueBytesFailureCase because it
602    // goes through the InputStream KeyValue API which can't produce a null buffer
603    try {
604      new KeyValue(null, 0, 0);
605    } catch (IllegalArgumentException iae) {
606      assertEquals("Invalid to have null byte array in KeyValue.", iae.getMessage());
607      return;
608    }
609    fail("Should have thrown an IllegalArgumentException when "
610      + "creating a KeyValue with a null buffer");
611  }
612
613  private static class FailureCase {
614    byte[] buf;
615    int offset;
616    int length;
617    boolean withTags;
618    String expectedMessage;
619
620    public FailureCase(byte[] buf, int offset, int length, boolean withTags,
621      String expectedMessage) {
622      this.buf = buf;
623      this.offset = offset;
624      this.length = length;
625      this.withTags = withTags;
626      this.expectedMessage = expectedMessage;
627    }
628
629    @Override
630    public String toString() {
631      return "FailureCaseDetails: [buf=" + Bytes.toStringBinary(buf, offset, length) + ", offset="
632        + offset + ", " + "length=" + length + ", expectedMessage=" + expectedMessage
633        + ", withtags=" + withTags + "]";
634    }
635
636    public String getExpectedMessage() {
637      return this.expectedMessage + KeyValueUtil.bytesToHex(buf, offset, length);
638    }
639  }
640
641  @Test
642  public void testCheckKeyValueBytesFailureCase() throws Exception {
643    byte[][] inputs = new byte[][] { HConstants.EMPTY_BYTE_ARRAY, // case.0
644      Bytes.toBytesBinary("a"), // case.1
645      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01"), // case.2
646      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00"), // case.3
647      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01"), // case.4
648      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00"), // case.5
649      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x01"), // case.6
650      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x03ROW"), // case.7
651      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01"), // case.8
652      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\xFF"
653        + "\\xFF\\xFF\\xFF\\xFF\\xFF\\xFF\\xFF\\x03"), // case.9
654      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
655        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x03"), // case.10
656      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
657        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04"), // case.11
658      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
659        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04VALUE"), // case.12
660    };
661    String[] outputs = new String[] { "Overflow when reading key length at position=0",
662      "Overflow when reading key length at position=0",
663      "Invalid key length in KeyValue. keyLength=1",
664      "Overflow when reading value length at position=4",
665      "Invalid value length in KeyValue, valueLength=1",
666      "Overflow when reading row length at position=8",
667      "Invalid row length in KeyValue, rowLength=1",
668      "Overflow when reading family length at position=13",
669      "Invalid family length in KeyValue, familyLength=1", "Timestamp cannot be negative, ts=-1",
670      "Invalid type in KeyValue, type=3", "Overflow when reading value part at position=25",
671      "Invalid tags length in KeyValue at position=26" };
672    byte[][] withTagsInputs = new byte[][] {
673      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
674        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x01"), // case.13
675      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
676        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x01"), // case.14
677      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
678        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x04\\x00\\x03\\x00A"), // case.15
679      // case.16
680      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
681        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0A\\x00\\x04\\x00TAG\\x00\\x04"
682        + "\\xFFT"),
683      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
684        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0C\\x00\\x04\\x00TAG\\x00\\x05"
685        + "\\xF0COME\\x00"), // case.17
686      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
687        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0C\\x00\\x04\\x00TAG\\x00\\x05"
688        + "\\xF0COME"), // case.18
689      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
690        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x00"), // case.19
691      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
692        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x1B\\x00\\x05\\x01TAG1\\x00\\x05"
693        + "\\x02TAG2\\x00\\x05\\x03TAG3\\x00\\x05\\x04TAG4"), // case.20
694    };
695    String[] withTagsOutputs = new String[] { "Overflow when reading tags length at position=26",
696      "Invalid tags length in KeyValue at position=26",
697      "Invalid tag length at position=28, tagLength=3",
698      "Invalid tag length at position=34, tagLength=4",
699      "Some redundant bytes in KeyValue's buffer, startOffset=41, endOffset=42", null, null,
700      null, };
701    assertEquals(inputs.length, outputs.length);
702    assertEquals(withTagsInputs.length, withTagsOutputs.length);
703
704    FailureCase[] cases = new FailureCase[inputs.length + withTagsInputs.length];
705    for (int i = 0; i < inputs.length; i++) {
706      cases[i] = new FailureCase(inputs[i], 0, inputs[i].length, false, outputs[i]);
707    }
708    for (int i = 0; i < withTagsInputs.length; i++) {
709      cases[inputs.length + i] =
710        new FailureCase(withTagsInputs[i], 0, withTagsInputs[i].length, true, withTagsOutputs[i]);
711    }
712
713    for (int i = 0; i < cases.length; i++) {
714      FailureCase c = cases[i];
715      ByteArrayOutputStream baos = new ByteArrayOutputStream();
716      DataOutputStream os = new DataOutputStream(baos);
717      ByteBufferUtils.putInt(os, c.length);
718      os.write(c.buf, c.offset, c.length);
719      try {
720        KeyValueUtil.createKeyValueFromInputStream(
721          new DataInputStream(new ByteArrayInputStream(baos.toByteArray())), c.withTags);
722        if (c.expectedMessage != null) {
723          fail("Should fail when parse key value from an invalid bytes for case#" + i + ". " + c);
724        }
725      } catch (IllegalArgumentException e) {
726        assertEquals("Case#" + i + " failed," + c, c.getExpectedMessage(), e.getMessage());
727      }
728    }
729  }
730}