Source code

001/**
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019package org.apache.hadoop.hbase;
020
021import static org.junit.Assert.assertArrayEquals;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertFalse;
024import static org.junit.Assert.assertNotEquals;
025import static org.junit.Assert.assertNotNull;
026import static org.junit.Assert.assertTrue;
027import static org.junit.Assert.fail;
028
029import java.io.ByteArrayInputStream;
030import java.io.ByteArrayOutputStream;
031import java.io.DataInputStream;
032import java.io.DataOutputStream;
033import java.util.Collections;
034import java.util.Iterator;
035import java.util.List;
036import java.util.Set;
037import java.util.TreeSet;
038
039import org.apache.hadoop.hbase.testclassification.SmallTests;
040import org.apache.hadoop.hbase.util.ByteBufferUtils;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.junit.ClassRule;
043import org.junit.Test;
044import org.junit.experimental.categories.Category;
045import org.slf4j.Logger;
046import org.slf4j.LoggerFactory;
047
048@Category(SmallTests.class)
049public class TestKeyValue {
050  @ClassRule
051  public static final HBaseClassTestRule CLASS_RULE =
052    HBaseClassTestRule.forClass(TestKeyValue.class);
053  private static final Logger LOG = LoggerFactory.getLogger(TestKeyValue.class);
054
055  @Test
056  public void testColumnCompare() {
057    final byte[] a = Bytes.toBytes("aaa");
058    byte[] family1 = Bytes.toBytes("abc");
059    byte[] qualifier1 = Bytes.toBytes("def");
060    byte[] family2 = Bytes.toBytes("abcd");
061    byte[] qualifier2 = Bytes.toBytes("ef");
062
063    KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, KeyValue.Type.Put, a);
064    assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
065    assertTrue(CellUtil.matchingColumn(aaa, family1, qualifier1));
066    aaa = new KeyValue(a, family2, qualifier2, 0L, KeyValue.Type.Put, a);
067    assertFalse(CellUtil.matchingColumn(aaa, family1, qualifier1));
068    assertTrue(CellUtil.matchingColumn(aaa, family2,qualifier2));
069    byte[] nullQualifier = new byte[0];
070    aaa = new KeyValue(a, family1, nullQualifier, 0L, KeyValue.Type.Put, a);
071    assertTrue(CellUtil.matchingColumn(aaa, family1,null));
072    assertFalse(CellUtil.matchingColumn(aaa, family2,qualifier2));
073  }
074
075  /**
076   * Test a corner case when the family qualifier is a prefix of the column qualifier.
077   */
078  @Test
079  public void testColumnCompare_prefix() {
080    final byte[] a = Bytes.toBytes("aaa");
081    byte[] family1 = Bytes.toBytes("abc");
082    byte[] qualifier1 = Bytes.toBytes("def");
083    byte[] family2 = Bytes.toBytes("ab");
084    byte[] qualifier2 = Bytes.toBytes("def");
085
086    KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, KeyValue.Type.Put, a);
087    assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
088  }
089
090  @Test
091  public void testBasics() {
092    LOG.info("LOWKEY: " + KeyValue.LOWESTKEY.toString());
093    String name = "testBasics";
094    check(Bytes.toBytes(name),
095      Bytes.toBytes(name), Bytes.toBytes(name), 1,
096      Bytes.toBytes(name));
097    // Test empty value and empty column -- both should work. (not empty fam)
098    check(Bytes.toBytes(name), Bytes.toBytes(name), null, 1, null);
099    check(HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes(name), null, 1, null);
100    // empty qual is equivalent to null qual
101    assertEquals(
102      new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"), null, 1, (byte[]) null),
103      new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"),
104        HConstants.EMPTY_BYTE_ARRAY, 1, (byte[]) null));
105  }
106
107  private void check(final byte [] row, final byte [] family, byte [] qualifier,
108    final long timestamp, final byte [] value) {
109    KeyValue kv = new KeyValue(row, family, qualifier, timestamp, value);
110    assertTrue(Bytes.compareTo(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), row, 0,
111      row.length) == 0);
112    assertTrue(CellUtil.matchingColumn(kv, family, qualifier));
113    // Call toString to make sure it works.
114    LOG.info(kv.toString());
115  }
116
117  @Test
118  public void testPlainCompare() {
119    final byte[] a = Bytes.toBytes("aaa");
120    final byte[] b = Bytes.toBytes("bbb");
121    final byte[] fam = Bytes.toBytes("col");
122    final byte[] qf = Bytes.toBytes("umn");
123    KeyValue aaa = new KeyValue(a, fam, qf, a);
124    KeyValue bbb = new KeyValue(b, fam, qf, b);
125    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) < 0);
126    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) > 0);
127    // Compare breaks if passed same ByteBuffer as both left and right arguments.
128    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, bbb) == 0);
129    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0);
130    // Do compare with different timestamps.
131    aaa = new KeyValue(a, fam, qf, 1, a);
132    bbb = new KeyValue(a, fam, qf, 2, a);
133    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) > 0);
134    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) < 0);
135    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0);
136    // Do compare with different types.  Higher numbered types -- Delete
137    // should sort ahead of lower numbers; i.e. Put
138    aaa = new KeyValue(a, fam, qf, 1, KeyValue.Type.Delete, a);
139    bbb = new KeyValue(a, fam, qf, 1, a);
140    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) < 0);
141    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) > 0);
142    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0);
143  }
144
145  @Test
146  public void testMoreComparisons() {
147    long now = System.currentTimeMillis();
148
149    // Meta compares
150    KeyValue aaa = new KeyValue(
151        Bytes.toBytes("TestScanMultipleVersions,row_0500,1236020145502"), now);
152    KeyValue bbb = new KeyValue(
153        Bytes.toBytes("TestScanMultipleVersions,,99999999999999"), now);
154    CellComparator c = MetaCellComparator.META_COMPARATOR;
155    assertTrue(c.compare(bbb, aaa) < 0);
156
157    KeyValue aaaa = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,,1236023996656"),
158        Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236024396271L,
159        (byte[])null);
160    assertTrue(c.compare(aaaa, bbb) < 0);
161
162    KeyValue x = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
163        Bytes.toBytes("info"), Bytes.toBytes(""), 9223372036854775807L,
164        (byte[])null);
165    KeyValue y = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
166        Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236034574912L,
167        (byte[])null);
168    assertTrue(c.compare(x, y) < 0);
169    comparisons(MetaCellComparator.META_COMPARATOR);
170    comparisons(CellComparatorImpl.COMPARATOR);
171    metacomparisons(MetaCellComparator.META_COMPARATOR);
172  }
173
174  @Test
175  public void testMetaComparatorTableKeysWithCommaOk() {
176    CellComparator c = MetaCellComparator.META_COMPARATOR;
177    long now = System.currentTimeMillis();
178    // meta keys values are not quite right.  A users can enter illegal values
179    // from shell when scanning meta.
180    KeyValue a = new KeyValue(Bytes.toBytes("table,key,with,commas1,1234"), now);
181    KeyValue b = new KeyValue(Bytes.toBytes("table,key,with,commas2,0123"), now);
182    assertTrue(c.compare(a, b) < 0);
183  }
184
185  /**
186   * Tests cases where rows keys have characters below the ','.
187   * See HBASE-832
188   */
189  @Test
190  public void testKeyValueBorderCases() {
191    // % sorts before , so if we don't do special comparator, rowB would
192    // come before rowA.
193    KeyValue rowA = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/,1234"),
194      Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
195    KeyValue rowB = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/%20,99999"),
196        Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
197    assertTrue(MetaCellComparator.META_COMPARATOR.compare(rowA, rowB) < 0);
198
199    rowA = new KeyValue(Bytes.toBytes("testtable,,1234"), Bytes.toBytes("fam"),
200        Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
201    rowB = new KeyValue(Bytes.toBytes("testtable,$www.hbase.org/,99999"),
202        Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
203    assertTrue(MetaCellComparator.META_COMPARATOR.compare(rowA, rowB) < 0);
204  }
205
206  private void metacomparisons(final CellComparatorImpl c) {
207    long now = System.currentTimeMillis();
208    assertTrue(c.compare(new KeyValue(
209        Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now),
210      new KeyValue(
211          Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now)) == 0);
212    KeyValue a = new KeyValue(
213        Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now);
214    KeyValue b = new KeyValue(
215        Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,2"), now);
216    assertTrue(c.compare(a, b) < 0);
217    assertTrue(c.compare(new KeyValue(
218        Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,2"), now),
219      new KeyValue(
220          Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now)) > 0);
221  }
222
223  private void comparisons(final CellComparatorImpl c) {
224    long now = System.currentTimeMillis();
225    assertTrue(c.compare(new KeyValue(
226        Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now),
227      new KeyValue(
228          Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now)) == 0);
229    assertTrue(c.compare(new KeyValue(
230        Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now),
231      new KeyValue(
232          Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,2"), now)) < 0);
233    assertTrue(c.compare(new KeyValue(
234        Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,2"), now),
235      new KeyValue(
236          Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now)) > 0);
237  }
238
239  @Test
240  public void testBinaryKeys() {
241    Set<KeyValue> set = new TreeSet<>(CellComparatorImpl.COMPARATOR);
242    final byte[] fam = Bytes.toBytes("col");
243    final byte[] qf = Bytes.toBytes("umn");
244    final byte[] nb = new byte[0];
245    KeyValue[] keys = {new KeyValue(Bytes.toBytes("aaaaa,\u0000\u0000,2"), fam, qf, 2, nb),
246      new KeyValue(Bytes.toBytes("aaaaa,\u0001,3"), fam, qf, 3, nb),
247      new KeyValue(Bytes.toBytes("aaaaa,,1"), fam, qf, 1, nb),
248      new KeyValue(Bytes.toBytes("aaaaa,\u1000,5"), fam, qf, 5, nb),
249      new KeyValue(Bytes.toBytes("aaaaa,a,4"), fam, qf, 4, nb),
250      new KeyValue(Bytes.toBytes("a,a,0"), fam, qf, 0, nb),
251    };
252    // Add to set with bad comparator
253    Collections.addAll(set, keys);
254    // This will output the keys incorrectly.
255    boolean assertion = false;
256    int count = 0;
257    try {
258      for (KeyValue k : set) {
259        assertEquals(count++, k.getTimestamp());
260      }
261    } catch (java.lang.AssertionError e) {
262      // Expected
263      assertion = true;
264    }
265    assertTrue(assertion);
266    // Make set with good comparator
267    set = new TreeSet<>(MetaCellComparator.META_COMPARATOR);
268    Collections.addAll(set, keys);
269    count = 0;
270    for (KeyValue k : set) {
271      assertEquals(count++, k.getTimestamp());
272    }
273  }
274
275  @Test
276  public void testStackedUpKeyValue() {
277    // Test multiple KeyValues in a single blob.
278
279    // TODO actually write this test!
280  }
281
282  private final byte[] rowA = Bytes.toBytes("rowA");
283  private final byte[] rowB = Bytes.toBytes("rowB");
284
285  private final byte[] family = Bytes.toBytes("family");
286  private final byte[] qualA = Bytes.toBytes("qfA");
287
288  private void assertKVLess(CellComparator c, KeyValue less, KeyValue greater) {
289    int cmp = c.compare(less,greater);
290    assertTrue(cmp < 0);
291    cmp = c.compare(greater,less);
292    assertTrue(cmp > 0);
293  }
294
295  private void assertKVLessWithoutRow(CellComparator c, KeyValue less, KeyValue greater) {
296    int cmp = c.compare(less, greater);
297    assertTrue(cmp < 0);
298    cmp = c.compare(greater, less);
299    assertTrue(cmp > 0);
300  }
301
302  @Test
303  public void testCompareWithoutRow() {
304    final CellComparator c = CellComparatorImpl.COMPARATOR;
305    byte[] row = Bytes.toBytes("row");
306
307    byte[] fa = Bytes.toBytes("fa");
308    byte[] fami = Bytes.toBytes("fami");
309    byte[] fami1 = Bytes.toBytes("fami1");
310
311    byte[] qual0 = Bytes.toBytes("");
312    byte[] qual1 = Bytes.toBytes("qf1");
313    byte[] qual2 = Bytes.toBytes("qf2");
314    long ts = 1;
315
316    // 'fa:'
317    KeyValue kv_0 = new KeyValue(row, fa, qual0, ts, KeyValue.Type.Put);
318    // 'fami:'
319    KeyValue kv0_0 = new KeyValue(row, fami, qual0, ts, KeyValue.Type.Put);
320    // 'fami:qf1'
321    KeyValue kv0_1 = new KeyValue(row, fami, qual1, ts, KeyValue.Type.Put);
322    // 'fami:qf2'
323    KeyValue kv0_2 = new KeyValue(row, fami, qual2, ts, KeyValue.Type.Put);
324    // 'fami1:'
325    KeyValue kv1_0 = new KeyValue(row, fami1, qual0, ts, KeyValue.Type.Put);
326
327    // 'fami:qf1' < 'fami:qf2'
328    assertKVLessWithoutRow(c, kv0_1, kv0_2);
329    // 'fami:qf1' < 'fami1:'
330    assertKVLessWithoutRow(c, kv0_1, kv1_0);
331
332    // Test comparison by skipping the same prefix bytes.
333    /*
334     * KeyValue Format and commonLength:
335     * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|....
336     * ------------------|-------commonLength--------|--------------
337     */
338    int commonLength = KeyValue.ROW_LENGTH_SIZE + KeyValue.FAMILY_LENGTH_SIZE
339        + row.length;
340    // 'fa:' < 'fami:'. They have commonPrefix + 2 same prefix bytes.
341    assertKVLessWithoutRow(c, kv_0, kv0_0);
342    // 'fami:' < 'fami:qf1'. They have commonPrefix + 4 same prefix bytes.
343    assertKVLessWithoutRow(c, kv0_0, kv0_1);
344    // 'fami:qf1' < 'fami1:'. They have commonPrefix + 4 same prefix bytes.
345    assertKVLessWithoutRow(c, kv0_1, kv1_0);
346    // 'fami:qf1' < 'fami:qf2'. They have commonPrefix + 6 same prefix bytes.
347    assertKVLessWithoutRow(c, kv0_1, kv0_2);
348  }
349
350  @Test
351  public void testFirstLastOnRow() {
352    final CellComparator c = CellComparatorImpl.COMPARATOR;
353    long ts = 1;
354    byte[] bufferA = new byte[128];
355    int offsetA = 0;
356    byte[] bufferB = new byte[128];
357    int offsetB = 7;
358
359    // These are listed in sort order (ie: every one should be less
360    // than the one on the next line).
361    final KeyValue firstOnRowA = KeyValueUtil.createFirstOnRow(rowA);
362    final KeyValue firstOnRowABufferFamQual = KeyValueUtil.createFirstOnRow(bufferA, offsetA,
363        rowA, 0, rowA.length, family, 0, family.length, qualA, 0, qualA.length);
364    final KeyValue kvA_1 = new KeyValue(rowA, null, null, ts, KeyValue.Type.Put);
365    final KeyValue kvA_2 = new KeyValue(rowA, family, qualA, ts, KeyValue.Type.Put);
366
367    final KeyValue lastOnRowA = KeyValueUtil.createLastOnRow(rowA);
368    final KeyValue firstOnRowB = KeyValueUtil.createFirstOnRow(rowB);
369    final KeyValue firstOnRowBBufferFam = KeyValueUtil.createFirstOnRow(bufferB, offsetB,
370        rowB, 0, rowB.length, family, 0, family.length, null, 0, 0);
371    final KeyValue kvB = new KeyValue(rowB, family, qualA, ts, KeyValue.Type.Put);
372
373    assertKVLess(c, firstOnRowA, firstOnRowB);
374    assertKVLess(c, firstOnRowA, firstOnRowBBufferFam);
375    assertKVLess(c, firstOnRowABufferFamQual, firstOnRowB);
376    assertKVLess(c, firstOnRowA, kvA_1);
377    assertKVLess(c, firstOnRowA, kvA_2);
378    assertKVLess(c, firstOnRowABufferFamQual, kvA_2);
379    assertKVLess(c, kvA_1, kvA_2);
380    assertKVLess(c, kvA_2, firstOnRowB);
381    assertKVLess(c, kvA_1, firstOnRowB);
382    assertKVLess(c, kvA_2, firstOnRowBBufferFam);
383    assertKVLess(c, kvA_1, firstOnRowBBufferFam);
384
385    assertKVLess(c, lastOnRowA, firstOnRowB);
386    assertKVLess(c, lastOnRowA, firstOnRowBBufferFam);
387    assertKVLess(c, firstOnRowB, kvB);
388    assertKVLess(c, firstOnRowBBufferFam, kvB);
389    assertKVLess(c, lastOnRowA, kvB);
390
391    assertKVLess(c, kvA_2, lastOnRowA);
392    assertKVLess(c, kvA_1, lastOnRowA);
393    assertKVLess(c, firstOnRowA, lastOnRowA);
394    assertKVLess(c, firstOnRowABufferFamQual, lastOnRowA);
395  }
396
397  @Test
398  public void testCreateKeyOnly() {
399    long ts = 1;
400    byte[] value = Bytes.toBytes("a real value");
401    byte[] evalue = new byte[0]; // empty value
402
403    for (byte[] val : new byte[][] { value, evalue }) {
404      for (boolean useLen : new boolean[]{false,true}) {
405        KeyValue kv1 = new KeyValue(rowA, family, qualA, ts, val);
406        KeyValue kv1ko = kv1.createKeyOnly(useLen);
407        // keys are still the same
408        assertTrue(kv1.equals(kv1ko));
409        // but values are not
410        assertTrue(kv1ko.getValueLength() == (useLen?Bytes.SIZEOF_INT:0));
411        if (useLen) {
412          assertEquals(kv1.getValueLength(),
413            Bytes.toInt(kv1ko.getValueArray(), kv1ko.getValueOffset(), kv1ko.getValueLength()));
414        }
415      }
416    }
417  }
418
419  @Test
420  public void testCreateKeyValueFromKey() {
421    KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
422        Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("myValue"));
423    int initialPadding = 10;
424    int endingPadding = 20;
425    int keyLen = kv.getKeyLength();
426    byte[] tmpArr = new byte[initialPadding + endingPadding + keyLen];
427    System.arraycopy(kv.getBuffer(), kv.getKeyOffset(), tmpArr,
428        initialPadding, keyLen);
429    KeyValue kvFromKey = KeyValueUtil.createKeyValueFromKey(tmpArr, initialPadding,
430        keyLen);
431    assertEquals(keyLen, kvFromKey.getKeyLength());
432    assertEquals(KeyValue.ROW_OFFSET + keyLen, kvFromKey.getBuffer().length);
433    System.err.println("kv=" + kv);
434    System.err.println("kvFromKey=" + kvFromKey);
435    assertEquals(kvFromKey.toString(),
436        kv.toString().replaceAll("=[0-9]+", "=0"));
437  }
438
439  /**
440   * Tests that getTimestamp() does always return the proper timestamp, even after updating it.
441   * See HBASE-6265.
442   */
443  @Test
444  public void testGetTimestamp() {
445    KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
446      Bytes.toBytes("myQualifier"), HConstants.LATEST_TIMESTAMP,
447      Bytes.toBytes("myValue"));
448    long time1 = kv.getTimestamp();
449    kv.updateLatestStamp(Bytes.toBytes(12345L));
450    long time2 = kv.getTimestamp();
451    assertEquals(HConstants.LATEST_TIMESTAMP, time1);
452    assertEquals(12345L, time2);
453  }
454
455  @Test
456  public void testKVsWithTags() {
457    byte[] row = Bytes.toBytes("myRow");
458    byte[] cf = Bytes.toBytes("myCF");
459    byte[] q = Bytes.toBytes("myQualifier");
460    byte[] value = Bytes.toBytes("myValue");
461    byte[] metaValue1 = Bytes.toBytes("metaValue1");
462    byte[] metaValue2 = Bytes.toBytes("metaValue2");
463    KeyValue kv = new KeyValue(row, cf, q, HConstants.LATEST_TIMESTAMP, value, new Tag[] {
464      new ArrayBackedTag((byte) 1, metaValue1), new ArrayBackedTag((byte) 2, metaValue2)
465    });
466    assertTrue(kv.getTagsLength() > 0);
467    assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), row, 0,
468      row.length));
469    assertTrue(Bytes.equals(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), cf, 0,
470      cf.length));
471    assertTrue(Bytes.equals(kv.getQualifierArray(), kv.getQualifierOffset(),
472      kv.getQualifierLength(), q, 0, q.length));
473    assertTrue(Bytes.equals(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength(), value, 0,
474      value.length));
475    List<Tag> tags = PrivateCellUtil.getTags(kv);
476    assertNotNull(tags);
477    assertEquals(2, tags.size());
478    boolean meta1Ok = false, meta2Ok = false;
479    for (Tag tag : tags) {
480      if (tag.getType() == (byte) 1) {
481        if (Bytes.equals(Tag.cloneValue(tag), metaValue1)) {
482          meta1Ok = true;
483        }
484      } else {
485        if (Bytes.equals(Tag.cloneValue(tag), metaValue2)) {
486          meta2Ok = true;
487        }
488      }
489    }
490    assertTrue(meta1Ok);
491    assertTrue(meta2Ok);
492    Iterator<Tag> tagItr = PrivateCellUtil.tagsIterator(kv);
493
494    assertTrue(tagItr.hasNext());
495    Tag next = tagItr.next();
496    assertEquals(10, next.getValueLength());
497    assertEquals((byte) 1, next.getType());
498    Bytes.equals(Tag.cloneValue(next), metaValue1);
499    assertTrue(tagItr.hasNext());
500    next = tagItr.next();
501    assertEquals(10, next.getValueLength());
502    assertEquals((byte) 2, next.getType());
503    Bytes.equals(Tag.cloneValue(next), metaValue2);
504    assertFalse(tagItr.hasNext());
505
506    tagItr = PrivateCellUtil.tagsIterator(kv);
507    assertTrue(tagItr.hasNext());
508    next = tagItr.next();
509    assertEquals(10, next.getValueLength());
510    assertEquals((byte) 1, next.getType());
511    Bytes.equals(Tag.cloneValue(next), metaValue1);
512    assertTrue(tagItr.hasNext());
513    next = tagItr.next();
514    assertEquals(10, next.getValueLength());
515    assertEquals((byte) 2, next.getType());
516    Bytes.equals(Tag.cloneValue(next), metaValue2);
517    assertFalse(tagItr.hasNext());
518  }
519
520  @Test
521  public void testMetaKeyComparator() {
522    CellComparator c = MetaCellComparator.META_COMPARATOR;
523    long now = System.currentTimeMillis();
524
525    KeyValue a = new KeyValue(Bytes.toBytes("table1"), now);
526    KeyValue b = new KeyValue(Bytes.toBytes("table2"), now);
527    assertTrue(c.compare(a, b) < 0);
528
529    a = new KeyValue(Bytes.toBytes("table1,111"), now);
530    b = new KeyValue(Bytes.toBytes("table2"), now);
531    assertTrue(c.compare(a, b) < 0);
532
533    a = new KeyValue(Bytes.toBytes("table1"), now);
534    b = new KeyValue(Bytes.toBytes("table2,111"), now);
535    assertTrue(c.compare(a, b) < 0);
536
537    a = new KeyValue(Bytes.toBytes("table,111"), now);
538    b = new KeyValue(Bytes.toBytes("table,2222"), now);
539    assertTrue(c.compare(a, b) < 0);
540
541    a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
542    b = new KeyValue(Bytes.toBytes("table,2222"), now);
543    assertTrue(c.compare(a, b) < 0);
544
545    a = new KeyValue(Bytes.toBytes("table,111"), now);
546    b = new KeyValue(Bytes.toBytes("table,2222.bbb"), now);
547    assertTrue(c.compare(a, b) < 0);
548
549    a = new KeyValue(Bytes.toBytes("table,,aaaa"), now);
550    b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
551    assertTrue(c.compare(a, b) < 0);
552
553    a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
554    b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
555    assertTrue(c.compare(a, b) < 0);
556
557    a = new KeyValue(Bytes.toBytes("table,111,xxxx"), now);
558    b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
559    assertTrue(c.compare(a, b) < 0);
560
561    a = new KeyValue(Bytes.toBytes("table,111,11,xxx"), now);
562    b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
563    assertTrue(c.compare(a, b) < 0);
564  }
565
566  @Test
567  public void testEqualsAndHashCode() {
568    KeyValue kvA1 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"),
569        Bytes.toBytes("qualA"), Bytes.toBytes("1"));
570    KeyValue kvA2 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"),
571        Bytes.toBytes("qualA"), Bytes.toBytes("2"));
572    // We set a different sequence id on kvA2 to demonstrate that the equals and hashCode also
573    // don't take this into account.
574    kvA2.setSequenceId(2);
575    KeyValue kvB = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"),
576        Bytes.toBytes("qualB"), Bytes.toBytes("1"));
577
578    assertEquals(kvA1, kvA2);
579    assertNotEquals(kvA1, kvB);
580    assertEquals(kvA1.hashCode(), kvA2.hashCode());
581    assertNotEquals(kvA1.hashCode(), kvB.hashCode());
582  }
583
584  @Test
585  public void testKeyValueSerialization() throws Exception {
586    KeyValue[] keyValues = new KeyValue[] {
587      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
588            Bytes.toBytes("1")),
589      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
590            Bytes.toBytes("2")),
591      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
592            System.currentTimeMillis(), Bytes.toBytes("2"),
593            new Tag[] {
594              new ArrayBackedTag((byte) 120, "tagA"),
595              new ArrayBackedTag((byte) 121, Bytes.toBytes("tagB")) }),
596      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
597            System.currentTimeMillis(), Bytes.toBytes("2"),
598            new Tag[] { new ArrayBackedTag((byte) 0, "tagA") }),
599      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes(""),
600            Bytes.toBytes("1")) };
601    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
602    for (KeyValue kv : keyValues) {
603      DataOutputStream os = new DataOutputStream(byteArrayOutputStream);
604      ByteBufferUtils.putInt(os, KeyValueUtil.getSerializedSize(kv, true));
605      KeyValueUtil.oswrite(kv, os, true);
606    }
607    DataInputStream is =
608        new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()));
609    for (int i = 0; i < keyValues.length; i++) {
610      LOG.info("Case#" + i + ": deserialize the kv: " + keyValues[i]);
611      KeyValue destKv = KeyValueUtil.createKeyValueFromInputStream(is, true);
612      assertEquals(keyValues[i], destKv);
613      assertArrayEquals(CellUtil.cloneValue(keyValues[i]), CellUtil.cloneValue(destKv));
614      assertArrayEquals(PrivateCellUtil.cloneTags(keyValues[i]), PrivateCellUtil.cloneTags(destKv));
615    }
616  }
617
618  @Test
619  public void testNullByteArrayKeyValueFailure() {
620    // can't add to testCheckKeyValueBytesFailureCase because it
621    // goes through the InputStream KeyValue API which can't produce a null buffer
622    try {
623      new KeyValue(null, 0, 0);
624    } catch (IllegalArgumentException iae) {
625      assertEquals("Invalid to have null byte array in KeyValue.", iae.getMessage());
626      return;
627    }
628    fail("Should have thrown an IllegalArgumentException when " +
629        "creating a KeyValue with a null buffer");
630  }
631
632  private static class FailureCase {
633    byte[] buf;
634    int offset;
635    int length;
636    boolean withTags;
637    String expectedMessage;
638
639    public FailureCase(byte[] buf, int offset, int length, boolean withTags,
640        String expectedMessage) {
641      this.buf = buf;
642      this.offset = offset;
643      this.length = length;
644      this.withTags = withTags;
645      this.expectedMessage = expectedMessage;
646    }
647
648    @Override
649    public String toString() {
650      return "FailureCaseDetails: [buf=" + Bytes.toStringBinary(buf, offset, length) + ", offset="
651          + offset + ", " + "length=" + length + ", expectedMessage=" + expectedMessage
652          + ", withtags=" + withTags + "]";
653    }
654
655    public String getExpectedMessage() {
656      return this.expectedMessage + KeyValueUtil.bytesToHex(buf, offset, length);
657    }
658  }
659
660  @Test
661  public void testCheckKeyValueBytesFailureCase() throws Exception {
662    byte[][] inputs = new byte[][] { HConstants.EMPTY_BYTE_ARRAY, // case.0
663      Bytes.toBytesBinary("a"), // case.1
664      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01"), // case.2
665      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00"), // case.3
666      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01"), // case.4
667      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00"), // case.5
668      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x01"), // case.6
669      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x03ROW"), // case.7
670      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01"), // case.8
671      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\xFF"
672          + "\\xFF\\xFF\\xFF\\xFF\\xFF\\xFF\\xFF\\x03"), // case.9
673      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
674          + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x03"), // case.10
675      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
676          + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04"), // case.11
677      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
678          + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04VALUE"), // case.12
679    };
680    String[] outputs = new String[] { "Overflow when reading key length at position=0",
681      "Overflow when reading key length at position=0",
682      "Invalid key length in KeyValue. keyLength=1",
683      "Overflow when reading value length at position=4",
684      "Invalid value length in KeyValue, valueLength=1",
685      "Overflow when reading row length at position=8",
686      "Invalid row length in KeyValue, rowLength=1",
687      "Overflow when reading family length at position=13",
688      "Invalid family length in KeyValue, familyLength=1", "Timestamp cannot be negative, ts=-1",
689      "Invalid type in KeyValue, type=3", "Overflow when reading value part at position=25",
690      "Invalid tags length in KeyValue at position=26"};
691    byte[][] withTagsInputs = new byte[][] {
692      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
693          + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x01"), // case.13
694      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
695          + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x01"), // case.14
696      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
697          + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x04\\x00\\x03\\x00A"), // case.15
698      // case.16
699      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
700          + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0A\\x00\\x04\\x00TAG\\x00\\x04"
701          + "\\xFFT"),
702      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
703          + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0C\\x00\\x04\\x00TAG\\x00\\x05"
704          + "\\xF0COME\\x00"), // case.17
705      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
706          + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0C\\x00\\x04\\x00TAG\\x00\\x05"
707          + "\\xF0COME"), // case.18
708      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
709          + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x00"), // case.19
710      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
711          + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x1B\\x00\\x05\\x01TAG1\\x00\\x05"
712          + "\\x02TAG2\\x00\\x05\\x03TAG3\\x00\\x05\\x04TAG4"), // case.20
713    };
714    String[] withTagsOutputs = new String[] { "Overflow when reading tags length at position=26",
715      "Invalid tags length in KeyValue at position=26",
716      "Invalid tag length at position=28, tagLength=3",
717      "Invalid tag length at position=34, tagLength=4",
718      "Some redundant bytes in KeyValue's buffer, startOffset=41, endOffset=42", null, null,
719      null,
720    };
721    assertEquals(inputs.length, outputs.length);
722    assertEquals(withTagsInputs.length, withTagsOutputs.length);
723
724    FailureCase[] cases = new FailureCase[inputs.length + withTagsInputs.length];
725    for (int i = 0; i < inputs.length; i++) {
726      cases[i] = new FailureCase(inputs[i], 0, inputs[i].length, false, outputs[i]);
727    }
728    for (int i = 0; i < withTagsInputs.length; i++) {
729      cases[inputs.length + i] =
730          new FailureCase(withTagsInputs[i], 0, withTagsInputs[i].length, true, withTagsOutputs[i]);
731    }
732
733    for (int i = 0; i < cases.length; i++) {
734      FailureCase c = cases[i];
735      ByteArrayOutputStream baos = new ByteArrayOutputStream();
736      DataOutputStream os = new DataOutputStream(baos);
737      ByteBufferUtils.putInt(os, c.length);
738      os.write(c.buf, c.offset, c.length);
739      try {
740        KeyValueUtil.createKeyValueFromInputStream(
741          new DataInputStream(new ByteArrayInputStream(baos.toByteArray())), c.withTags);
742        if (c.expectedMessage != null) {
743          fail("Should fail when parse kv from an invalid bytes, case#" + i + ". " + c);
744        }
745      } catch (IllegalArgumentException e) {
746        assertEquals("Case#" + i + " failed," + c, c.getExpectedMessage(), e.getMessage());
747      }
748    }
749  }
750}