Source code

001/**
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019package org.apache.hadoop.hbase;
020
021import static org.junit.Assert.assertArrayEquals;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertFalse;
024import static org.junit.Assert.assertNotEquals;
025import static org.junit.Assert.assertNotNull;
026import static org.junit.Assert.assertTrue;
027
028import java.io.ByteArrayInputStream;
029import java.io.ByteArrayOutputStream;
030import java.io.DataInputStream;
031import java.io.DataOutputStream;
032import java.io.IOException;
033import java.util.Collections;
034import java.util.Iterator;
035import java.util.List;
036import java.util.Set;
037import java.util.TreeSet;
038
039import org.apache.hadoop.hbase.testclassification.SmallTests;
040import org.apache.hadoop.hbase.util.ByteBufferUtils;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.junit.Assert;
043import org.junit.ClassRule;
044import org.junit.Test;
045import org.junit.experimental.categories.Category;
046import org.slf4j.Logger;
047import org.slf4j.LoggerFactory;
048
049
050@Category(SmallTests.class)
051public class TestKeyValue {
052  @ClassRule
053  public static final HBaseClassTestRule CLASS_RULE =
054    HBaseClassTestRule.forClass(TestKeyValue.class);
055  private static final Logger LOG = LoggerFactory.getLogger(TestKeyValue.class);
056
057  @Test
058  public void testColumnCompare() throws Exception {
059    final byte [] a = Bytes.toBytes("aaa");
060    byte [] family1 = Bytes.toBytes("abc");
061    byte [] qualifier1 = Bytes.toBytes("def");
062    byte [] family2 = Bytes.toBytes("abcd");
063    byte [] qualifier2 = Bytes.toBytes("ef");
064
065    KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, KeyValue.Type.Put, a);
066    assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
067    assertTrue(CellUtil.matchingColumn(aaa, family1, qualifier1));
068    aaa = new KeyValue(a, family2, qualifier2, 0L, KeyValue.Type.Put, a);
069    assertFalse(CellUtil.matchingColumn(aaa, family1, qualifier1));
070    assertTrue(CellUtil.matchingColumn(aaa, family2,qualifier2));
071    byte [] nullQualifier = new byte[0];
072    aaa = new KeyValue(a, family1, nullQualifier, 0L, KeyValue.Type.Put, a);
073    assertTrue(CellUtil.matchingColumn(aaa, family1,null));
074    assertFalse(CellUtil.matchingColumn(aaa, family2,qualifier2));
075  }
076
077  /**
078   * Test a corner case when the family qualifier is a prefix of the
079   *  column qualifier.
080   */
081  @Test
082  public void testColumnCompare_prefix() throws Exception {
083    final byte [] a = Bytes.toBytes("aaa");
084    byte [] family1 = Bytes.toBytes("abc");
085    byte [] qualifier1 = Bytes.toBytes("def");
086    byte [] family2 = Bytes.toBytes("ab");
087    byte [] qualifier2 = Bytes.toBytes("def");
088
089    KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, KeyValue.Type.Put, a);
090    assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
091  }
092
093  @Test
094  public void testBasics() throws Exception {
095    LOG.info("LOWKEY: " + KeyValue.LOWESTKEY.toString());
096    String name = "testBasics";
097    check(Bytes.toBytes(name),
098      Bytes.toBytes(name), Bytes.toBytes(name), 1,
099      Bytes.toBytes(name));
100    // Test empty value and empty column -- both should work. (not empty fam)
101    check(Bytes.toBytes(name), Bytes.toBytes(name), null, 1, null);
102    check(HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes(name), null, 1, null);
103    // empty qual is equivalent to null qual
104    assertEquals(
105      new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"), null, 1, (byte[]) null),
106      new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"),
107        HConstants.EMPTY_BYTE_ARRAY, 1, (byte[]) null));
108  }
109
110  private void check(final byte [] row, final byte [] family, byte [] qualifier,
111    final long timestamp, final byte [] value) {
112    KeyValue kv = new KeyValue(row, family, qualifier, timestamp, value);
113    assertTrue(Bytes.compareTo(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), row, 0,
114      row.length) == 0);
115    assertTrue(CellUtil.matchingColumn(kv, family, qualifier));
116    // Call toString to make sure it works.
117    LOG.info(kv.toString());
118  }
119
120  @Test
121  public void testPlainCompare() throws Exception {
122    final byte [] a = Bytes.toBytes("aaa");
123    final byte [] b = Bytes.toBytes("bbb");
124    final byte [] fam = Bytes.toBytes("col");
125    final byte [] qf = Bytes.toBytes("umn");
126    KeyValue aaa = new KeyValue(a, fam, qf, a);
127    KeyValue bbb = new KeyValue(b, fam, qf, b);
128    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) < 0);
129    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) > 0);
130    // Compare breaks if passed same ByteBuffer as both left and right arguments.
131    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, bbb) == 0);
132    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0);
133    // Do compare with different timestamps.
134    aaa = new KeyValue(a, fam, qf, 1, a);
135    bbb = new KeyValue(a, fam, qf, 2, a);
136    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) > 0);
137    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) < 0);
138    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0);
139    // Do compare with different types.  Higher numbered types -- Delete
140    // should sort ahead of lower numbers; i.e. Put
141    aaa = new KeyValue(a, fam, qf, 1, KeyValue.Type.Delete, a);
142    bbb = new KeyValue(a, fam, qf, 1, a);
143    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) < 0);
144    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) > 0);
145    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0);
146  }
147
148  @Test
149  public void testMoreComparisons() throws Exception {
150    long now = System.currentTimeMillis();
151
152    // Meta compares
153    KeyValue aaa = new KeyValue(
154        Bytes.toBytes("TestScanMultipleVersions,row_0500,1236020145502"), now);
155    KeyValue bbb = new KeyValue(
156        Bytes.toBytes("TestScanMultipleVersions,,99999999999999"), now);
157    CellComparator c = CellComparatorImpl.META_COMPARATOR;
158    assertTrue(c.compare(bbb, aaa) < 0);
159
160    KeyValue aaaa = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,,1236023996656"),
161        Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236024396271L,
162        (byte[])null);
163    assertTrue(c.compare(aaaa, bbb) < 0);
164
165    KeyValue x = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
166        Bytes.toBytes("info"), Bytes.toBytes(""), 9223372036854775807L,
167        (byte[])null);
168    KeyValue y = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
169        Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236034574912L,
170        (byte[])null);
171    assertTrue(c.compare(x, y) < 0);
172    comparisons(CellComparatorImpl.META_COMPARATOR);
173    comparisons(CellComparatorImpl.COMPARATOR);
174    metacomparisons(CellComparatorImpl.META_COMPARATOR);
175  }
176
177  @Test
178  public void testMetaComparatorTableKeysWithCommaOk() {
179    CellComparator c = CellComparatorImpl.META_COMPARATOR;
180    long now = System.currentTimeMillis();
181    // meta keys values are not quite right.  A users can enter illegal values
182    // from shell when scanning meta.
183    KeyValue a = new KeyValue(Bytes.toBytes("table,key,with,commas1,1234"), now);
184    KeyValue b = new KeyValue(Bytes.toBytes("table,key,with,commas2,0123"), now);
185    assertTrue(c.compare(a, b) < 0);
186  }
187
188  /**
189   * Tests cases where rows keys have characters below the ','.
190   * See HBASE-832
191   * @throws IOException
192   */
193  @Test
194  public void testKeyValueBorderCases() throws IOException {
195    // % sorts before , so if we don't do special comparator, rowB would
196    // come before rowA.
197    KeyValue rowA = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/,1234"),
198      Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
199    KeyValue rowB = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/%20,99999"),
200        Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
201    assertTrue(CellComparatorImpl.META_COMPARATOR.compare(rowA, rowB) < 0);
202
203    rowA = new KeyValue(Bytes.toBytes("testtable,,1234"), Bytes.toBytes("fam"),
204        Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
205    rowB = new KeyValue(Bytes.toBytes("testtable,$www.hbase.org/,99999"),
206        Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
207    assertTrue(CellComparatorImpl.META_COMPARATOR.compare(rowA, rowB) < 0);
208
209  }
210
211  private void metacomparisons(final CellComparatorImpl c) {
212    long now = System.currentTimeMillis();
213    assertTrue(c.compare(new KeyValue(
214        Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now),
215      new KeyValue(
216          Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now)) == 0);
217    KeyValue a = new KeyValue(
218        Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now);
219    KeyValue b = new KeyValue(
220        Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,2"), now);
221    assertTrue(c.compare(a, b) < 0);
222    assertTrue(c.compare(new KeyValue(
223        Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,2"), now),
224      new KeyValue(
225          Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now)) > 0);
226  }
227
228  private void comparisons(final CellComparatorImpl c) {
229    long now = System.currentTimeMillis();
230    assertTrue(c.compare(new KeyValue(
231        Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now),
232      new KeyValue(
233          Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now)) == 0);
234    assertTrue(c.compare(new KeyValue(
235        Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now),
236      new KeyValue(
237          Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,2"), now)) < 0);
238    assertTrue(c.compare(new KeyValue(
239        Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,2"), now),
240      new KeyValue(
241          Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now)) > 0);
242  }
243
244  @Test
245  public void testBinaryKeys() throws Exception {
246    Set<KeyValue> set = new TreeSet<>(CellComparatorImpl.COMPARATOR);
247    final byte [] fam = Bytes.toBytes("col");
248    final byte [] qf = Bytes.toBytes("umn");
249    final byte [] nb = new byte[0];
250    KeyValue [] keys = {new KeyValue(Bytes.toBytes("aaaaa,\u0000\u0000,2"), fam, qf, 2, nb),
251      new KeyValue(Bytes.toBytes("aaaaa,\u0001,3"), fam, qf, 3, nb),
252      new KeyValue(Bytes.toBytes("aaaaa,,1"), fam, qf, 1, nb),
253      new KeyValue(Bytes.toBytes("aaaaa,\u1000,5"), fam, qf, 5, nb),
254      new KeyValue(Bytes.toBytes("aaaaa,a,4"), fam, qf, 4, nb),
255      new KeyValue(Bytes.toBytes("a,a,0"), fam, qf, 0, nb),
256    };
257    // Add to set with bad comparator
258    Collections.addAll(set, keys);
259    // This will output the keys incorrectly.
260    boolean assertion = false;
261    int count = 0;
262    try {
263      for (KeyValue k: set) {
264        assertTrue(count++ == k.getTimestamp());
265      }
266    } catch (java.lang.AssertionError e) {
267      // Expected
268      assertion = true;
269    }
270    assertTrue(assertion);
271    // Make set with good comparator
272    set = new TreeSet<>(CellComparatorImpl.META_COMPARATOR);
273    Collections.addAll(set, keys);
274    count = 0;
275    for (KeyValue k: set) {
276      assertTrue(count++ == k.getTimestamp());
277    }
278  }
279
280  @Test
281  public void testStackedUpKeyValue() {
282    // Test multiple KeyValues in a single blob.
283
284    // TODO actually write this test!
285
286  }
287
288  private final byte[] rowA = Bytes.toBytes("rowA");
289  private final byte[] rowB = Bytes.toBytes("rowB");
290
291  private final byte[] family = Bytes.toBytes("family");
292  private final byte[] qualA = Bytes.toBytes("qfA");
293  private final byte[] qualB = Bytes.toBytes("qfB");
294
295  private void assertKVLess(CellComparator c,
296                            KeyValue less,
297                            KeyValue greater) {
298    int cmp = c.compare(less,greater);
299    assertTrue(cmp < 0);
300    cmp = c.compare(greater,less);
301    assertTrue(cmp > 0);
302  }
303
304  private void assertKVLessWithoutRow(CellComparator c, int common, KeyValue less,
305      KeyValue greater) {
306    int cmp = c.compare(less, greater);
307    assertTrue(cmp < 0);
308    cmp = c.compare(greater, less);
309    assertTrue(cmp > 0);
310  }
311
312  @Test
313  public void testCompareWithoutRow() {
314    final CellComparator c = CellComparatorImpl.COMPARATOR;
315    byte[] row = Bytes.toBytes("row");
316
317    byte[] fa = Bytes.toBytes("fa");
318    byte[] fami = Bytes.toBytes("fami");
319    byte[] fami1 = Bytes.toBytes("fami1");
320
321    byte[] qual0 = Bytes.toBytes("");
322    byte[] qual1 = Bytes.toBytes("qf1");
323    byte[] qual2 = Bytes.toBytes("qf2");
324    long ts = 1;
325
326    // 'fa:'
327    KeyValue kv_0 = new KeyValue(row, fa, qual0, ts, KeyValue.Type.Put);
328    // 'fami:'
329    KeyValue kv0_0 = new KeyValue(row, fami, qual0, ts, KeyValue.Type.Put);
330    // 'fami:qf1'
331    KeyValue kv0_1 = new KeyValue(row, fami, qual1, ts, KeyValue.Type.Put);
332    // 'fami:qf2'
333    KeyValue kv0_2 = new KeyValue(row, fami, qual2, ts, KeyValue.Type.Put);
334    // 'fami1:'
335    KeyValue kv1_0 = new KeyValue(row, fami1, qual0, ts, KeyValue.Type.Put);
336
337    // 'fami:qf1' < 'fami:qf2'
338    assertKVLessWithoutRow(c, 0, kv0_1, kv0_2);
339    // 'fami:qf1' < 'fami1:'
340    assertKVLessWithoutRow(c, 0, kv0_1, kv1_0);
341
342    // Test comparison by skipping the same prefix bytes.
343    /***
344     * KeyValue Format and commonLength:
345     * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|....
346     * ------------------|-------commonLength--------|--------------
347     */
348    int commonLength = KeyValue.ROW_LENGTH_SIZE + KeyValue.FAMILY_LENGTH_SIZE
349        + row.length;
350    // 'fa:' < 'fami:'. They have commonPrefix + 2 same prefix bytes.
351    assertKVLessWithoutRow(c, commonLength + 2, kv_0, kv0_0);
352    // 'fami:' < 'fami:qf1'. They have commonPrefix + 4 same prefix bytes.
353    assertKVLessWithoutRow(c, commonLength + 4, kv0_0, kv0_1);
354    // 'fami:qf1' < 'fami1:'. They have commonPrefix + 4 same prefix bytes.
355    assertKVLessWithoutRow(c, commonLength + 4, kv0_1, kv1_0);
356    // 'fami:qf1' < 'fami:qf2'. They have commonPrefix + 6 same prefix bytes.
357    assertKVLessWithoutRow(c, commonLength + 6, kv0_1, kv0_2);
358  }
359
360  @Test
361  public void testFirstLastOnRow() {
362    final CellComparator c = CellComparatorImpl.COMPARATOR;
363    long ts = 1;
364    byte[] bufferA = new byte[128];
365    int offsetA = 0;
366    byte[] bufferB = new byte[128];
367    int offsetB = 7;
368
369    // These are listed in sort order (ie: every one should be less
370    // than the one on the next line).
371    final KeyValue firstOnRowA = KeyValueUtil.createFirstOnRow(rowA);
372    final KeyValue firstOnRowABufferFamQual = KeyValueUtil.createFirstOnRow(bufferA, offsetA,
373        rowA, 0, rowA.length, family, 0, family.length, qualA, 0, qualA.length);
374    final KeyValue kvA_1 = new KeyValue(rowA, null, null, ts, KeyValue.Type.Put);
375    final KeyValue kvA_2 = new KeyValue(rowA, family, qualA, ts, KeyValue.Type.Put);
376
377    final KeyValue lastOnRowA = KeyValueUtil.createLastOnRow(rowA);
378    final KeyValue firstOnRowB = KeyValueUtil.createFirstOnRow(rowB);
379    final KeyValue firstOnRowBBufferFam = KeyValueUtil.createFirstOnRow(bufferB, offsetB,
380        rowB, 0, rowB.length, family, 0, family.length, null, 0, 0);
381    final KeyValue kvB = new KeyValue(rowB, family, qualA, ts, KeyValue.Type.Put);
382
383    assertKVLess(c, firstOnRowA, firstOnRowB);
384    assertKVLess(c, firstOnRowA, firstOnRowBBufferFam);
385    assertKVLess(c, firstOnRowABufferFamQual, firstOnRowB);
386    assertKVLess(c, firstOnRowA, kvA_1);
387    assertKVLess(c, firstOnRowA, kvA_2);
388    assertKVLess(c, firstOnRowABufferFamQual, kvA_2);
389    assertKVLess(c, kvA_1, kvA_2);
390    assertKVLess(c, kvA_2, firstOnRowB);
391    assertKVLess(c, kvA_1, firstOnRowB);
392    assertKVLess(c, kvA_2, firstOnRowBBufferFam);
393    assertKVLess(c, kvA_1, firstOnRowBBufferFam);
394
395    assertKVLess(c, lastOnRowA, firstOnRowB);
396    assertKVLess(c, lastOnRowA, firstOnRowBBufferFam);
397    assertKVLess(c, firstOnRowB, kvB);
398    assertKVLess(c, firstOnRowBBufferFam, kvB);
399    assertKVLess(c, lastOnRowA, kvB);
400
401    assertKVLess(c, kvA_2, lastOnRowA);
402    assertKVLess(c, kvA_1, lastOnRowA);
403    assertKVLess(c, firstOnRowA, lastOnRowA);
404    assertKVLess(c, firstOnRowABufferFamQual, lastOnRowA);
405  }
406
407  @Test
408  public void testCreateKeyOnly() throws Exception {
409    long ts = 1;
410    byte [] value = Bytes.toBytes("a real value");
411    byte [] evalue = new byte[0]; // empty value
412
413    for (byte[] val : new byte[][]{value, evalue}) {
414      for (boolean useLen : new boolean[]{false,true}) {
415        KeyValue kv1 = new KeyValue(rowA, family, qualA, ts, val);
416        KeyValue kv1ko = kv1.createKeyOnly(useLen);
417        // keys are still the same
418        assertTrue(kv1.equals(kv1ko));
419        // but values are not
420        assertTrue(kv1ko.getValueLength() == (useLen?Bytes.SIZEOF_INT:0));
421        if (useLen) {
422          assertEquals(kv1.getValueLength(),
423            Bytes.toInt(kv1ko.getValueArray(), kv1ko.getValueOffset(), kv1ko.getValueLength()));
424        }
425      }
426    }
427  }
428
429  @Test
430  public void testCreateKeyValueFromKey() {
431    KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
432        Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("myValue"));
433    int initialPadding = 10;
434    int endingPadding = 20;
435    int keyLen = kv.getKeyLength();
436    byte[] tmpArr = new byte[initialPadding + endingPadding + keyLen];
437    System.arraycopy(kv.getBuffer(), kv.getKeyOffset(), tmpArr,
438        initialPadding, keyLen);
439    KeyValue kvFromKey = KeyValueUtil.createKeyValueFromKey(tmpArr, initialPadding,
440        keyLen);
441    assertEquals(keyLen, kvFromKey.getKeyLength());
442    assertEquals(KeyValue.ROW_OFFSET + keyLen, kvFromKey.getBuffer().length);
443    System.err.println("kv=" + kv);
444    System.err.println("kvFromKey=" + kvFromKey);
445    assertEquals(kvFromKey.toString(),
446        kv.toString().replaceAll("=[0-9]+", "=0"));
447  }
448
449  /**
450   * Tests that getTimestamp() does always return the proper timestamp, even after updating it.
451   * See HBASE-6265.
452   */
453  @Test
454  public void testGetTimestamp() {
455    KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
456      Bytes.toBytes("myQualifier"), HConstants.LATEST_TIMESTAMP,
457      Bytes.toBytes("myValue"));
458    long time1 = kv.getTimestamp();
459    kv.updateLatestStamp(Bytes.toBytes(12345L));
460    long time2 = kv.getTimestamp();
461    assertEquals(HConstants.LATEST_TIMESTAMP, time1);
462    assertEquals(12345L, time2);
463  }
464
465  @Test
466  public void testKVsWithTags() {
467    byte[] row = Bytes.toBytes("myRow");
468    byte[] cf = Bytes.toBytes("myCF");
469    byte[] q = Bytes.toBytes("myQualifier");
470    byte[] value = Bytes.toBytes("myValue");
471    byte[] metaValue1 = Bytes.toBytes("metaValue1");
472    byte[] metaValue2 = Bytes.toBytes("metaValue2");
473    KeyValue kv = new KeyValue(row, cf, q, HConstants.LATEST_TIMESTAMP, value, new Tag[] {
474        new ArrayBackedTag((byte) 1, metaValue1), new ArrayBackedTag((byte) 2, metaValue2) });
475    assertTrue(kv.getTagsLength() > 0);
476    assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), row, 0,
477      row.length));
478    assertTrue(Bytes.equals(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), cf, 0,
479      cf.length));
480    assertTrue(Bytes.equals(kv.getQualifierArray(), kv.getQualifierOffset(),
481      kv.getQualifierLength(), q, 0, q.length));
482    assertTrue(Bytes.equals(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength(), value, 0,
483      value.length));
484    List<Tag> tags = PrivateCellUtil.getTags(kv);
485    assertNotNull(tags);
486    assertEquals(2, tags.size());
487    boolean meta1Ok = false, meta2Ok = false;
488    for (Tag tag : tags) {
489      if (tag.getType() == (byte) 1) {
490        if (Bytes.equals(Tag.cloneValue(tag), metaValue1)) {
491          meta1Ok = true;
492        }
493      } else {
494        if (Bytes.equals(Tag.cloneValue(tag), metaValue2)) {
495          meta2Ok = true;
496        }
497      }
498    }
499    assertTrue(meta1Ok);
500    assertTrue(meta2Ok);
501    Iterator<Tag> tagItr = PrivateCellUtil.tagsIterator(kv);
502    //Iterator<Tag> tagItr = kv.tagsIterator();
503    assertTrue(tagItr.hasNext());
504    Tag next = tagItr.next();
505    assertEquals(10, next.getValueLength());
506    assertEquals((byte) 1, next.getType());
507    Bytes.equals(Tag.cloneValue(next), metaValue1);
508    assertTrue(tagItr.hasNext());
509    next = tagItr.next();
510    assertEquals(10, next.getValueLength());
511    assertEquals((byte) 2, next.getType());
512    Bytes.equals(Tag.cloneValue(next), metaValue2);
513    assertFalse(tagItr.hasNext());
514
515    tagItr = PrivateCellUtil.tagsIterator(kv);
516    assertTrue(tagItr.hasNext());
517    next = tagItr.next();
518    assertEquals(10, next.getValueLength());
519    assertEquals((byte) 1, next.getType());
520    Bytes.equals(Tag.cloneValue(next), metaValue1);
521    assertTrue(tagItr.hasNext());
522    next = tagItr.next();
523    assertEquals(10, next.getValueLength());
524    assertEquals((byte) 2, next.getType());
525    Bytes.equals(Tag.cloneValue(next), metaValue2);
526    assertFalse(tagItr.hasNext());
527  }
528
529  @Test
530  public void testMetaKeyComparator() {
531    CellComparator c = CellComparatorImpl.META_COMPARATOR;
532    long now = System.currentTimeMillis();
533
534    KeyValue a = new KeyValue(Bytes.toBytes("table1"), now);
535    KeyValue b = new KeyValue(Bytes.toBytes("table2"), now);
536    assertTrue(c.compare(a, b) < 0);
537
538    a = new KeyValue(Bytes.toBytes("table1,111"), now);
539    b = new KeyValue(Bytes.toBytes("table2"), now);
540    assertTrue(c.compare(a, b) < 0);
541
542    a = new KeyValue(Bytes.toBytes("table1"), now);
543    b = new KeyValue(Bytes.toBytes("table2,111"), now);
544    assertTrue(c.compare(a, b) < 0);
545
546    a = new KeyValue(Bytes.toBytes("table,111"), now);
547    b = new KeyValue(Bytes.toBytes("table,2222"), now);
548    assertTrue(c.compare(a, b) < 0);
549
550    a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
551    b = new KeyValue(Bytes.toBytes("table,2222"), now);
552    assertTrue(c.compare(a, b) < 0);
553
554    a = new KeyValue(Bytes.toBytes("table,111"), now);
555    b = new KeyValue(Bytes.toBytes("table,2222.bbb"), now);
556    assertTrue(c.compare(a, b) < 0);
557
558    a = new KeyValue(Bytes.toBytes("table,,aaaa"), now);
559    b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
560    assertTrue(c.compare(a, b) < 0);
561
562    a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
563    b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
564    assertTrue(c.compare(a, b) < 0);
565
566    a = new KeyValue(Bytes.toBytes("table,111,xxxx"), now);
567    b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
568    assertTrue(c.compare(a, b) < 0);
569
570    a = new KeyValue(Bytes.toBytes("table,111,11,xxx"), now);
571    b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
572    assertTrue(c.compare(a, b) < 0);
573  }
574
575  @Test
576  public void testEqualsAndHashCode() throws Exception {
577    KeyValue kvA1 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"),
578        Bytes.toBytes("qualA"), Bytes.toBytes("1"));
579    KeyValue kvA2 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"),
580        Bytes.toBytes("qualA"), Bytes.toBytes("2"));
581    // We set a different sequence id on kvA2 to demonstrate that the equals and hashCode also
582    // don't take this into account.
583    kvA2.setSequenceId(2);
584    KeyValue kvB = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"),
585        Bytes.toBytes("qualB"), Bytes.toBytes("1"));
586
587    assertEquals(kvA1, kvA2);
588    assertNotEquals(kvA1, kvB);
589    assertEquals(kvA1.hashCode(), kvA2.hashCode());
590    assertNotEquals(kvA1.hashCode(), kvB.hashCode());
591
592  }
593
594  @Test
595  public void testKeyValueSerialization() throws Exception {
596    KeyValue[] keyValues = new KeyValue[] {
597        new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
598            Bytes.toBytes("1")),
599        new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
600            Bytes.toBytes("2")),
601        new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
602            System.currentTimeMillis(), Bytes.toBytes("2"),
603            new Tag[] { new ArrayBackedTag((byte) 120, "tagA"),
604                new ArrayBackedTag((byte) 121, Bytes.toBytes("tagB")) }),
605        new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
606            System.currentTimeMillis(), Bytes.toBytes("2"),
607            new Tag[] { new ArrayBackedTag((byte) 0, "tagA") }),
608        new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes(""),
609            Bytes.toBytes("1")) };
610    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
611    for (KeyValue kv : keyValues) {
612      DataOutputStream os = new DataOutputStream(byteArrayOutputStream);
613      ByteBufferUtils.putInt(os, KeyValueUtil.getSerializedSize(kv, true));
614      KeyValueUtil.oswrite(kv, os, true);
615    }
616    DataInputStream is =
617        new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()));
618    for (int i = 0; i < keyValues.length; i++) {
619      LOG.info("Case#" + i + ": deserialize the kv: " + keyValues[i]);
620      KeyValue destKv = KeyValueUtil.createKeyValueFromInputStream(is, true);
621      assertEquals(keyValues[i], destKv);
622      assertArrayEquals(CellUtil.cloneValue(keyValues[i]), CellUtil.cloneValue(destKv));
623      assertArrayEquals(PrivateCellUtil.cloneTags(keyValues[i]), PrivateCellUtil.cloneTags(destKv));
624    }
625  }
626
627  private static class FailureCase {
628    byte[] buf;
629    int offset;
630    int length;
631    boolean withTags;
632    String expectedMessage;
633
634    public FailureCase(byte[] buf, int offset, int length, boolean withTags,
635        String expectedMessage) {
636      this.buf = buf;
637      this.offset = offset;
638      this.length = length;
639      this.withTags = withTags;
640      this.expectedMessage = expectedMessage;
641    }
642
643    @Override
644    public String toString() {
645      return "FailureCaseDetails: [buf=" + Bytes.toStringBinary(buf, offset, length) + ", offset="
646          + offset + ", " + "length=" + length + ", expectedMessage=" + expectedMessage
647          + ", withtags=" + withTags + "]";
648    }
649
650    public String getExpectedMessage() {
651      return this.expectedMessage + KeyValueUtil.bytesToHex(buf, offset, length);
652    }
653  }
654
655  @Test
656  public void testCheckKeyValueBytesFailureCase() throws Exception {
657    byte[][] inputs = new byte[][] { HConstants.EMPTY_BYTE_ARRAY, // case.0
658        Bytes.toBytesBinary("a"), // case.1
659        Bytes.toBytesBinary("\\x00\\x00\\x00\\x01"), // case.2
660        Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00"), // case.3
661        Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01"), // case.4
662        Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00"), // case.5
663        Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x01"), // case.6
664        Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x03ROW"), // case.7
665        Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01"), // case.8
666        Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\xFF"
667            + "\\xFF\\xFF\\xFF\\xFF\\xFF\\xFF\\xFF\\x03"), // case.9
668        Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
669            + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x03"), // case.10
670        Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
671            + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04"), // case.11
672        Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
673            + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04VALUE"), // case.12
674    };
675    String[] outputs = new String[] { "Overflow when reading key length at position=0",
676        "Overflow when reading key length at position=0",
677        "Invalid key length in KeyValue. keyLength=1",
678        "Overflow when reading value length at position=4",
679        "Invalid value length in KeyValue, valueLength=1",
680        "Overflow when reading row length at position=8",
681        "Invalid row length in KeyValue, rowLength=1",
682        "Overflow when reading family length at position=13",
683        "Invalid family length in KeyValue, familyLength=1", "Timestamp cannot be negative, ts=-1",
684        "Invalid type in KeyValue, type=3", "Overflow when reading value part at position=25",
685        "Invalid tags length in KeyValue at position=26", };
686    byte[][] withTagsInputs = new byte[][] {
687        Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
688            + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x01"), // case.13
689        Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
690            + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x01"), // case.14
691        Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
692            + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x04\\x00\\x03\\x00A"), // case.15
693        // case.16
694        Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
695            + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0A\\x00\\x04\\x00TAG\\x00\\x04"
696            + "\\xFFT"),
697        Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
698            + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0C\\x00\\x04\\x00TAG\\x00\\x05"
699            + "\\xF0COME\\x00"), // case.17
700        Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
701            + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0C\\x00\\x04\\x00TAG\\x00\\x05"
702            + "\\xF0COME"), // case.18
703        Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
704            + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x00"), // case.19
705        Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
706            + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x1B\\x00\\x05\\x01TAG1\\x00\\x05"
707            + "\\x02TAG2\\x00\\x05\\x03TAG3\\x00\\x05\\x04TAG4"), // case.20
708    };
709    String[] withTagsOutputs = new String[] { "Overflow when reading tags length at position=26",
710        "Invalid tags length in KeyValue at position=26",
711        "Invalid tag length at position=28, tagLength=3",
712        "Invalid tag length at position=34, tagLength=4",
713        "Some redundant bytes in KeyValue's buffer, startOffset=41, endOffset=42", null, null,
714        null, };
715    assertEquals(inputs.length, outputs.length);
716    assertEquals(withTagsInputs.length, withTagsOutputs.length);
717
718    FailureCase[] cases = new FailureCase[inputs.length + withTagsInputs.length];
719    for (int i = 0; i < inputs.length; i++) {
720      cases[i] = new FailureCase(inputs[i], 0, inputs[i].length, false, outputs[i]);
721    }
722    for (int i = 0; i < withTagsInputs.length; i++) {
723      cases[inputs.length + i] =
724          new FailureCase(withTagsInputs[i], 0, withTagsInputs[i].length, true, withTagsOutputs[i]);
725    }
726
727    for (int i = 0; i < cases.length; i++) {
728      FailureCase c = cases[i];
729      ByteArrayOutputStream baos = new ByteArrayOutputStream();
730      DataOutputStream os = new DataOutputStream(baos);
731      ByteBufferUtils.putInt(os, c.length);
732      os.write(c.buf, c.offset, c.length);
733      try {
734        KeyValueUtil.createKeyValueFromInputStream(
735          new DataInputStream(new ByteArrayInputStream(baos.toByteArray())), c.withTags);
736        if (c.expectedMessage != null) {
737          Assert.fail("Should fail when parse kv from an invalid bytes, case#" + i + ". " + c);
738        }
739      } catch (IllegalArgumentException e) {
740        assertEquals("Case#" + i + " failed," + c, c.getExpectedMessage(), e.getMessage());
741      }
742    }
743  }
744}