Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.jupiter.api.Assertions.assertArrayEquals;
021import static org.junit.jupiter.api.Assertions.assertEquals;
022import static org.junit.jupiter.api.Assertions.assertFalse;
023import static org.junit.jupiter.api.Assertions.assertNotEquals;
024import static org.junit.jupiter.api.Assertions.assertNotNull;
025import static org.junit.jupiter.api.Assertions.assertTrue;
026import static org.junit.jupiter.api.Assertions.fail;
027
028import java.io.ByteArrayInputStream;
029import java.io.ByteArrayOutputStream;
030import java.io.DataInputStream;
031import java.io.DataOutputStream;
032import java.util.Collections;
033import java.util.Iterator;
034import java.util.List;
035import java.util.Set;
036import java.util.TreeSet;
037import org.apache.hadoop.hbase.testclassification.MiscTests;
038import org.apache.hadoop.hbase.testclassification.SmallTests;
039import org.apache.hadoop.hbase.util.ByteBufferUtils;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
042import org.junit.jupiter.api.Test;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046@org.junit.jupiter.api.Tag(MiscTests.TAG)
047@org.junit.jupiter.api.Tag(SmallTests.TAG)
048public class TestKeyValue {
049  private static final Logger LOG = LoggerFactory.getLogger(TestKeyValue.class);
050
051  @Test
052  public void testColumnCompare() {
053    final byte[] a = Bytes.toBytes("aaa");
054    byte[] family1 = Bytes.toBytes("abc");
055    byte[] qualifier1 = Bytes.toBytes("def");
056    byte[] family2 = Bytes.toBytes("abcd");
057    byte[] qualifier2 = Bytes.toBytes("ef");
058
059    KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, KeyValue.Type.Put, a);
060    assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
061    assertTrue(CellUtil.matchingColumn(aaa, family1, qualifier1));
062    aaa = new KeyValue(a, family2, qualifier2, 0L, KeyValue.Type.Put, a);
063    assertFalse(CellUtil.matchingColumn(aaa, family1, qualifier1));
064    assertTrue(CellUtil.matchingColumn(aaa, family2, qualifier2));
065    byte[] nullQualifier = new byte[0];
066    aaa = new KeyValue(a, family1, nullQualifier, 0L, KeyValue.Type.Put, a);
067    assertTrue(CellUtil.matchingColumn(aaa, family1, null));
068    assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
069  }
070
071  /**
072   * Test a corner case when the family qualifier is a prefix of the column qualifier.
073   */
074  @Test
075  public void testColumnCompare_prefix() {
076    final byte[] a = Bytes.toBytes("aaa");
077    byte[] family1 = Bytes.toBytes("abc");
078    byte[] qualifier1 = Bytes.toBytes("def");
079    byte[] family2 = Bytes.toBytes("ab");
080    byte[] qualifier2 = Bytes.toBytes("def");
081
082    KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, KeyValue.Type.Put, a);
083    assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
084  }
085
086  @Test
087  public void testBasics() {
088    LOG.info("LOWKEY: " + KeyValue.LOWESTKEY.toString());
089    String name = "testBasics";
090    check(Bytes.toBytes(name), Bytes.toBytes(name), Bytes.toBytes(name), 1, Bytes.toBytes(name));
091    // Test empty value and empty column -- both should work. (not empty fam)
092    check(Bytes.toBytes(name), Bytes.toBytes(name), null, 1, null);
093    check(HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes(name), null, 1, null);
094    // empty qual is equivalent to null qual
095    assertEquals(new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"), null, 1, (byte[]) null),
096      new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"), HConstants.EMPTY_BYTE_ARRAY, 1,
097        (byte[]) null));
098  }
099
100  private void check(final byte[] row, final byte[] family, byte[] qualifier, final long timestamp,
101    final byte[] value) {
102    KeyValue kv = new KeyValue(row, family, qualifier, timestamp, value);
103    assertTrue(
104      Bytes.compareTo(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), row, 0, row.length)
105          == 0);
106    assertTrue(CellUtil.matchingColumn(kv, family, qualifier));
107    // Call toString to make sure it works.
108    LOG.info(kv.toString());
109  }
110
111  @Test
112  public void testPlainCompare() {
113    final byte[] a = Bytes.toBytes("aaa");
114    final byte[] b = Bytes.toBytes("bbb");
115    final byte[] fam = Bytes.toBytes("col");
116    final byte[] qf = Bytes.toBytes("umn");
117    KeyValue aaa = new KeyValue(a, fam, qf, a);
118    KeyValue bbb = new KeyValue(b, fam, qf, b);
119    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) < 0);
120    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) > 0);
121    // Compare breaks if passed same ByteBuffer as both left and right arguments.
122    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, bbb) == 0);
123    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0);
124    // Do compare with different timestamps.
125    aaa = new KeyValue(a, fam, qf, 1, a);
126    bbb = new KeyValue(a, fam, qf, 2, a);
127    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) > 0);
128    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) < 0);
129    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0);
130    // Do compare with different types. Higher numbered types -- Delete
131    // should sort ahead of lower numbers; i.e. Put
132    aaa = new KeyValue(a, fam, qf, 1, KeyValue.Type.Delete, a);
133    bbb = new KeyValue(a, fam, qf, 1, a);
134    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) < 0);
135    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) > 0);
136    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0);
137  }
138
139  @Test
140  public void testMoreComparisons() {
141    long now = EnvironmentEdgeManager.currentTime();
142
143    // Meta compares
144    KeyValue aaa =
145      new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236020145502"), now);
146    KeyValue bbb = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,,99999999999999"), now);
147    CellComparator c = MetaCellComparator.META_COMPARATOR;
148    assertTrue(c.compare(bbb, aaa) < 0);
149
150    KeyValue aaaa = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,,1236023996656"),
151      Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236024396271L, (byte[]) null);
152    assertTrue(c.compare(aaaa, bbb) < 0);
153
154    KeyValue x = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
155      Bytes.toBytes("info"), Bytes.toBytes(""), 9223372036854775807L, (byte[]) null);
156    KeyValue y = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
157      Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236034574912L, (byte[]) null);
158    assertTrue(c.compare(x, y) < 0);
159    comparisons(MetaCellComparator.META_COMPARATOR);
160    comparisons(CellComparatorImpl.COMPARATOR);
161    metacomparisons(MetaCellComparator.META_COMPARATOR);
162  }
163
164  @Test
165  public void testMetaComparatorTableKeysWithCommaOk() {
166    CellComparator c = MetaCellComparator.META_COMPARATOR;
167    long now = EnvironmentEdgeManager.currentTime();
168    // meta keys values are not quite right. A users can enter illegal values
169    // from shell when scanning meta.
170    KeyValue a = new KeyValue(Bytes.toBytes("table,key,with,commas1,1234"), now);
171    KeyValue b = new KeyValue(Bytes.toBytes("table,key,with,commas2,0123"), now);
172    assertTrue(c.compare(a, b) < 0);
173  }
174
175  /**
176   * Tests cases where rows keys have characters below the ','. See HBASE-832
177   */
178  @Test
179  public void testKeyValueBorderCases() {
180    // % sorts before , so if we don't do special comparator, rowB would
181    // come before rowA.
182    KeyValue rowA = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/,1234"),
183      Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[]) null);
184    KeyValue rowB = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/%20,99999"),
185      Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[]) null);
186    assertTrue(MetaCellComparator.META_COMPARATOR.compare(rowA, rowB) < 0);
187
188    rowA = new KeyValue(Bytes.toBytes("testtable,,1234"), Bytes.toBytes("fam"), Bytes.toBytes(""),
189      Long.MAX_VALUE, (byte[]) null);
190    rowB = new KeyValue(Bytes.toBytes("testtable,$www.hbase.org/,99999"), Bytes.toBytes("fam"),
191      Bytes.toBytes(""), Long.MAX_VALUE, (byte[]) null);
192    assertTrue(MetaCellComparator.META_COMPARATOR.compare(rowA, rowB) < 0);
193  }
194
195  private void metacomparisons(final CellComparatorImpl c) {
196    long now = EnvironmentEdgeManager.currentTime();
197    assertTrue(c.compare(
198      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now),
199      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now))
200        == 0);
201    KeyValue a =
202      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now);
203    KeyValue b =
204      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,2"), now);
205    assertTrue(c.compare(a, b) < 0);
206    assertTrue(c.compare(
207      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,2"), now),
208      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now))
209        > 0);
210  }
211
212  private void comparisons(final CellComparatorImpl c) {
213    long now = EnvironmentEdgeManager.currentTime();
214    assertTrue(c.compare(
215      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now),
216      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now)) == 0);
217    assertTrue(c.compare(
218      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now),
219      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,2"), now)) < 0);
220    assertTrue(c.compare(
221      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,2"), now),
222      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now)) > 0);
223  }
224
225  @Test
226  public void testBinaryKeys() {
227    Set<KeyValue> set = new TreeSet<>(CellComparatorImpl.COMPARATOR);
228    final byte[] fam = Bytes.toBytes("col");
229    final byte[] qf = Bytes.toBytes("umn");
230    final byte[] nb = new byte[0];
231    KeyValue[] keys = { new KeyValue(Bytes.toBytes("aaaaa,\u0000\u0000,2"), fam, qf, 2, nb),
232      new KeyValue(Bytes.toBytes("aaaaa,\u0001,3"), fam, qf, 3, nb),
233      new KeyValue(Bytes.toBytes("aaaaa,,1"), fam, qf, 1, nb),
234      new KeyValue(Bytes.toBytes("aaaaa,\u1000,5"), fam, qf, 5, nb),
235      new KeyValue(Bytes.toBytes("aaaaa,a,4"), fam, qf, 4, nb),
236      new KeyValue(Bytes.toBytes("a,a,0"), fam, qf, 0, nb), };
237    // Add to set with bad comparator
238    Collections.addAll(set, keys);
239    // This will output the keys incorrectly.
240    boolean assertion = false;
241    int count = 0;
242    for (KeyValue k : set) {
243      if (count++ != k.getTimestamp()) {
244        assertion = true;
245      }
246    }
247    assertTrue(assertion);
248    // Make set with good comparator
249    set = new TreeSet<>(MetaCellComparator.META_COMPARATOR);
250    Collections.addAll(set, keys);
251    count = 0;
252    for (KeyValue k : set) {
253      assertEquals(count++, k.getTimestamp());
254    }
255  }
256
257  @Test
258  public void testStackedUpKeyValue() {
259    // Test multiple KeyValues in a single blob.
260
261    // TODO actually write this test!
262  }
263
264  private final byte[] rowA = Bytes.toBytes("rowA");
265  private final byte[] rowB = Bytes.toBytes("rowB");
266
267  private final byte[] family = Bytes.toBytes("family");
268  private final byte[] qualA = Bytes.toBytes("qfA");
269
270  private void assertKVLess(CellComparator c, KeyValue less, KeyValue greater) {
271    int cmp = c.compare(less, greater);
272    assertTrue(cmp < 0);
273    cmp = c.compare(greater, less);
274    assertTrue(cmp > 0);
275  }
276
277  private void assertKVLessWithoutRow(CellComparator c, KeyValue less, KeyValue greater) {
278    int cmp = c.compare(less, greater);
279    assertTrue(cmp < 0);
280    cmp = c.compare(greater, less);
281    assertTrue(cmp > 0);
282  }
283
284  @Test
285  public void testCompareWithoutRow() {
286    final CellComparator c = CellComparatorImpl.COMPARATOR;
287    byte[] row = Bytes.toBytes("row");
288
289    byte[] fa = Bytes.toBytes("fa");
290    byte[] fami = Bytes.toBytes("fami");
291    byte[] fami1 = Bytes.toBytes("fami1");
292
293    byte[] qual0 = Bytes.toBytes("");
294    byte[] qual1 = Bytes.toBytes("qf1");
295    byte[] qual2 = Bytes.toBytes("qf2");
296    long ts = 1;
297
298    // 'fa:'
299    KeyValue kv_0 = new KeyValue(row, fa, qual0, ts, KeyValue.Type.Put);
300    // 'fami:'
301    KeyValue kv0_0 = new KeyValue(row, fami, qual0, ts, KeyValue.Type.Put);
302    // 'fami:qf1'
303    KeyValue kv0_1 = new KeyValue(row, fami, qual1, ts, KeyValue.Type.Put);
304    // 'fami:qf2'
305    KeyValue kv0_2 = new KeyValue(row, fami, qual2, ts, KeyValue.Type.Put);
306    // 'fami1:'
307    KeyValue kv1_0 = new KeyValue(row, fami1, qual0, ts, KeyValue.Type.Put);
308
309    // 'fami:qf1' < 'fami:qf2'
310    assertKVLessWithoutRow(c, kv0_1, kv0_2);
311    // 'fami:qf1' < 'fami1:'
312    assertKVLessWithoutRow(c, kv0_1, kv1_0);
313
314    // Test comparison by skipping the same prefix bytes.
315    /*
316     * KeyValue Format and commonLength:
317     * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|....
318     * ------------------|-------commonLength--------|--------------
319     */
320    // 'fa:' < 'fami:'. They have commonPrefix + 2 same prefix bytes.
321    assertKVLessWithoutRow(c, kv_0, kv0_0);
322    // 'fami:' < 'fami:qf1'. They have commonPrefix + 4 same prefix bytes.
323    assertKVLessWithoutRow(c, kv0_0, kv0_1);
324    // 'fami:qf1' < 'fami1:'. They have commonPrefix + 4 same prefix bytes.
325    assertKVLessWithoutRow(c, kv0_1, kv1_0);
326    // 'fami:qf1' < 'fami:qf2'. They have commonPrefix + 6 same prefix bytes.
327    assertKVLessWithoutRow(c, kv0_1, kv0_2);
328  }
329
330  @Test
331  public void testFirstLastOnRow() {
332    final CellComparator c = CellComparatorImpl.COMPARATOR;
333    long ts = 1;
334    byte[] bufferA = new byte[128];
335    int offsetA = 0;
336    byte[] bufferB = new byte[128];
337    int offsetB = 7;
338
339    // These are listed in sort order (ie: every one should be less
340    // than the one on the next line).
341    final KeyValue firstOnRowA = KeyValueUtil.createFirstOnRow(rowA);
342    final KeyValue firstOnRowABufferFamQual = KeyValueUtil.createFirstOnRow(bufferA, offsetA, rowA,
343      0, rowA.length, family, 0, family.length, qualA, 0, qualA.length);
344    final KeyValue kvA_1 = new KeyValue(rowA, null, null, ts, KeyValue.Type.Put);
345    final KeyValue kvA_2 = new KeyValue(rowA, family, qualA, ts, KeyValue.Type.Put);
346
347    final KeyValue lastOnRowA = KeyValueUtil.createLastOnRow(rowA);
348    final KeyValue firstOnRowB = KeyValueUtil.createFirstOnRow(rowB);
349    final KeyValue firstOnRowBBufferFam = KeyValueUtil.createFirstOnRow(bufferB, offsetB, rowB, 0,
350      rowB.length, family, 0, family.length, null, 0, 0);
351    final KeyValue kvB = new KeyValue(rowB, family, qualA, ts, KeyValue.Type.Put);
352
353    assertKVLess(c, firstOnRowA, firstOnRowB);
354    assertKVLess(c, firstOnRowA, firstOnRowBBufferFam);
355    assertKVLess(c, firstOnRowABufferFamQual, firstOnRowB);
356    assertKVLess(c, firstOnRowA, kvA_1);
357    assertKVLess(c, firstOnRowA, kvA_2);
358    assertKVLess(c, firstOnRowABufferFamQual, kvA_2);
359    assertKVLess(c, kvA_1, kvA_2);
360    assertKVLess(c, kvA_2, firstOnRowB);
361    assertKVLess(c, kvA_1, firstOnRowB);
362    assertKVLess(c, kvA_2, firstOnRowBBufferFam);
363    assertKVLess(c, kvA_1, firstOnRowBBufferFam);
364
365    assertKVLess(c, lastOnRowA, firstOnRowB);
366    assertKVLess(c, lastOnRowA, firstOnRowBBufferFam);
367    assertKVLess(c, firstOnRowB, kvB);
368    assertKVLess(c, firstOnRowBBufferFam, kvB);
369    assertKVLess(c, lastOnRowA, kvB);
370
371    assertKVLess(c, kvA_2, lastOnRowA);
372    assertKVLess(c, kvA_1, lastOnRowA);
373    assertKVLess(c, firstOnRowA, lastOnRowA);
374    assertKVLess(c, firstOnRowABufferFamQual, lastOnRowA);
375  }
376
377  @Test
378  public void testCreateKeyOnly() {
379    long ts = 1;
380    byte[] value = Bytes.toBytes("a real value");
381    byte[] evalue = new byte[0]; // empty value
382
383    for (byte[] val : new byte[][] { value, evalue }) {
384      for (boolean useLen : new boolean[] { false, true }) {
385        KeyValue kv1 = new KeyValue(rowA, family, qualA, ts, val);
386        KeyValue kv1ko = kv1.createKeyOnly(useLen);
387        // keys are still the same
388        assertTrue(kv1.equals(kv1ko));
389        // but values are not
390        assertTrue(kv1ko.getValueLength() == (useLen ? Bytes.SIZEOF_INT : 0));
391        if (useLen) {
392          assertEquals(kv1.getValueLength(),
393            Bytes.toInt(kv1ko.getValueArray(), kv1ko.getValueOffset(), kv1ko.getValueLength()));
394        }
395      }
396    }
397  }
398
399  @Test
400  public void testCreateKeyValueFromKey() {
401    KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
402      Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("myValue"));
403    int initialPadding = 10;
404    int endingPadding = 20;
405    int keyLen = kv.getKeyLength();
406    byte[] tmpArr = new byte[initialPadding + endingPadding + keyLen];
407    System.arraycopy(kv.getBuffer(), kv.getKeyOffset(), tmpArr, initialPadding, keyLen);
408    KeyValue kvFromKey = KeyValueUtil.createKeyValueFromKey(tmpArr, initialPadding, keyLen);
409    assertEquals(keyLen, kvFromKey.getKeyLength());
410    assertEquals(KeyValue.ROW_OFFSET + keyLen, kvFromKey.getBuffer().length);
411    System.err.println("kv=" + kv);
412    System.err.println("kvFromKey=" + kvFromKey);
413    assertEquals(kvFromKey.toString(), kv.toString().replaceAll("=[0-9]+", "=0"));
414  }
415
416  /**
417   * Tests that getTimestamp() does always return the proper timestamp, even after updating it. See
418   * HBASE-6265.
419   */
420  @Test
421  public void testGetTimestamp() {
422    KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
423      Bytes.toBytes("myQualifier"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("myValue"));
424    long time1 = kv.getTimestamp();
425    kv.updateLatestStamp(Bytes.toBytes(12345L));
426    long time2 = kv.getTimestamp();
427    assertEquals(HConstants.LATEST_TIMESTAMP, time1);
428    assertEquals(12345L, time2);
429  }
430
431  @Test
432  public void testKVsWithTags() {
433    byte[] row = Bytes.toBytes("myRow");
434    byte[] cf = Bytes.toBytes("myCF");
435    byte[] q = Bytes.toBytes("myQualifier");
436    byte[] value = Bytes.toBytes("myValue");
437    byte[] metaValue1 = Bytes.toBytes("metaValue1");
438    byte[] metaValue2 = Bytes.toBytes("metaValue2");
439    KeyValue kv = new KeyValue(row, cf, q, HConstants.LATEST_TIMESTAMP, value, new Tag[] {
440      new ArrayBackedTag((byte) 1, metaValue1), new ArrayBackedTag((byte) 2, metaValue2) });
441    assertTrue(kv.getTagsLength() > 0);
442    assertTrue(
443      Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), row, 0, row.length));
444    assertTrue(Bytes.equals(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), cf, 0,
445      cf.length));
446    assertTrue(Bytes.equals(kv.getQualifierArray(), kv.getQualifierOffset(),
447      kv.getQualifierLength(), q, 0, q.length));
448    assertTrue(Bytes.equals(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength(), value, 0,
449      value.length));
450    List<Tag> tags = PrivateCellUtil.getTags(kv);
451    assertNotNull(tags);
452    assertEquals(2, tags.size());
453    boolean meta1Ok = false, meta2Ok = false;
454    for (Tag tag : tags) {
455      if (tag.getType() == (byte) 1) {
456        if (Bytes.equals(Tag.cloneValue(tag), metaValue1)) {
457          meta1Ok = true;
458        }
459      } else {
460        if (Bytes.equals(Tag.cloneValue(tag), metaValue2)) {
461          meta2Ok = true;
462        }
463      }
464    }
465    assertTrue(meta1Ok);
466    assertTrue(meta2Ok);
467    Iterator<Tag> tagItr = PrivateCellUtil.tagsIterator(kv);
468
469    assertTrue(tagItr.hasNext());
470    Tag next = tagItr.next();
471    assertEquals(10, next.getValueLength());
472    assertEquals((byte) 1, next.getType());
473    Bytes.equals(Tag.cloneValue(next), metaValue1);
474    assertTrue(tagItr.hasNext());
475    next = tagItr.next();
476    assertEquals(10, next.getValueLength());
477    assertEquals((byte) 2, next.getType());
478    Bytes.equals(Tag.cloneValue(next), metaValue2);
479    assertFalse(tagItr.hasNext());
480
481    tagItr = PrivateCellUtil.tagsIterator(kv);
482    assertTrue(tagItr.hasNext());
483    next = tagItr.next();
484    assertEquals(10, next.getValueLength());
485    assertEquals((byte) 1, next.getType());
486    Bytes.equals(Tag.cloneValue(next), metaValue1);
487    assertTrue(tagItr.hasNext());
488    next = tagItr.next();
489    assertEquals(10, next.getValueLength());
490    assertEquals((byte) 2, next.getType());
491    Bytes.equals(Tag.cloneValue(next), metaValue2);
492    assertFalse(tagItr.hasNext());
493  }
494
495  @Test
496  public void testMetaKeyComparator() {
497    CellComparator c = MetaCellComparator.META_COMPARATOR;
498    long now = EnvironmentEdgeManager.currentTime();
499
500    KeyValue a = new KeyValue(Bytes.toBytes("table1"), now);
501    KeyValue b = new KeyValue(Bytes.toBytes("table2"), now);
502    assertTrue(c.compare(a, b) < 0);
503
504    a = new KeyValue(Bytes.toBytes("table1,111"), now);
505    b = new KeyValue(Bytes.toBytes("table2"), now);
506    assertTrue(c.compare(a, b) < 0);
507
508    a = new KeyValue(Bytes.toBytes("table1"), now);
509    b = new KeyValue(Bytes.toBytes("table2,111"), now);
510    assertTrue(c.compare(a, b) < 0);
511
512    a = new KeyValue(Bytes.toBytes("table,111"), now);
513    b = new KeyValue(Bytes.toBytes("table,2222"), now);
514    assertTrue(c.compare(a, b) < 0);
515
516    a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
517    b = new KeyValue(Bytes.toBytes("table,2222"), now);
518    assertTrue(c.compare(a, b) < 0);
519
520    a = new KeyValue(Bytes.toBytes("table,111"), now);
521    b = new KeyValue(Bytes.toBytes("table,2222.bbb"), now);
522    assertTrue(c.compare(a, b) < 0);
523
524    a = new KeyValue(Bytes.toBytes("table,,aaaa"), now);
525    b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
526    assertTrue(c.compare(a, b) < 0);
527
528    a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
529    b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
530    assertTrue(c.compare(a, b) < 0);
531
532    a = new KeyValue(Bytes.toBytes("table,111,xxxx"), now);
533    b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
534    assertTrue(c.compare(a, b) < 0);
535
536    a = new KeyValue(Bytes.toBytes("table,111,11,xxx"), now);
537    b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
538    assertTrue(c.compare(a, b) < 0);
539  }
540
541  @Test
542  public void testEqualsAndHashCode() {
543    KeyValue kvA1 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
544      Bytes.toBytes("1"));
545    KeyValue kvA2 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
546      Bytes.toBytes("2"));
547    // We set a different sequence id on kvA2 to demonstrate that the equals and hashCode also
548    // don't take this into account.
549    kvA2.setSequenceId(2);
550    KeyValue kvB = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualB"),
551      Bytes.toBytes("1"));
552
553    assertEquals(kvA1, kvA2);
554    assertNotEquals(kvA1, kvB);
555    assertEquals(kvA1.hashCode(), kvA2.hashCode());
556    assertNotEquals(kvA1.hashCode(), kvB.hashCode());
557  }
558
559  @Test
560  public void testKeyValueSerialization() throws Exception {
561    KeyValue[] keyValues = new KeyValue[] {
562      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
563        Bytes.toBytes("1")),
564      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
565        Bytes.toBytes("2")),
566      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
567        EnvironmentEdgeManager.currentTime(), Bytes.toBytes("2"),
568        new Tag[] { new ArrayBackedTag((byte) 120, "tagA"),
569          new ArrayBackedTag((byte) 121, Bytes.toBytes("tagB")) }),
570      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
571        EnvironmentEdgeManager.currentTime(), Bytes.toBytes("2"),
572        new Tag[] { new ArrayBackedTag((byte) 0, "tagA") }),
573      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes(""),
574        Bytes.toBytes("1")) };
575    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
576    for (KeyValue kv : keyValues) {
577      DataOutputStream os = new DataOutputStream(byteArrayOutputStream);
578      ByteBufferUtils.putInt(os, kv.getSerializedSize(true));
579      kv.write(os, true);
580    }
581    DataInputStream is =
582      new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()));
583    for (int i = 0; i < keyValues.length; i++) {
584      LOG.info("Case#" + i + ": deserialize the kv: " + keyValues[i]);
585      KeyValue destKv = KeyValueUtil.createKeyValueFromInputStream(is, true);
586      assertEquals(keyValues[i], destKv);
587      assertArrayEquals(CellUtil.cloneValue(keyValues[i]), CellUtil.cloneValue(destKv));
588      assertArrayEquals(PrivateCellUtil.cloneTags(keyValues[i]), PrivateCellUtil.cloneTags(destKv));
589    }
590  }
591
592  @Test
593  public void testNullByteArrayKeyValueFailure() {
594    // can't add to testCheckKeyValueBytesFailureCase because it
595    // goes through the InputStream KeyValue API which can't produce a null buffer
596    try {
597      new KeyValue(null, 0, 0);
598    } catch (IllegalArgumentException iae) {
599      assertEquals("Invalid to have null byte array in KeyValue.", iae.getMessage());
600      return;
601    }
602    fail("Should have thrown an IllegalArgumentException when "
603      + "creating a KeyValue with a null buffer");
604  }
605
606  private static class FailureCase {
607    byte[] buf;
608    int offset;
609    int length;
610    boolean withTags;
611    String expectedMessage;
612
613    public FailureCase(byte[] buf, int offset, int length, boolean withTags,
614      String expectedMessage) {
615      this.buf = buf;
616      this.offset = offset;
617      this.length = length;
618      this.withTags = withTags;
619      this.expectedMessage = expectedMessage;
620    }
621
622    @Override
623    public String toString() {
624      return "FailureCaseDetails: [buf=" + Bytes.toStringBinary(buf, offset, length) + ", offset="
625        + offset + ", " + "length=" + length + ", expectedMessage=" + expectedMessage
626        + ", withtags=" + withTags + "]";
627    }
628
629    public String getExpectedMessage() {
630      return this.expectedMessage + KeyValueUtil.bytesToHex(buf, offset, length);
631    }
632  }
633
634  @Test
635  public void testCheckKeyValueBytesFailureCase() throws Exception {
636    byte[][] inputs = new byte[][] { HConstants.EMPTY_BYTE_ARRAY, // case.0
637      Bytes.toBytesBinary("a"), // case.1
638      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01"), // case.2
639      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00"), // case.3
640      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01"), // case.4
641      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00"), // case.5
642      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x01"), // case.6
643      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x03ROW"), // case.7
644      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01"), // case.8
645      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\xFF"
646        + "\\xFF\\xFF\\xFF\\xFF\\xFF\\xFF\\xFF\\x03"), // case.9
647      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
648        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x03"), // case.10
649      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
650        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04"), // case.11
651      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
652        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04VALUE"), // case.12
653    };
654    String[] outputs = new String[] { "Overflow when reading key length at position=0",
655      "Overflow when reading key length at position=0",
656      "Invalid key length in KeyValue. keyLength=1",
657      "Overflow when reading value length at position=4",
658      "Invalid value length in KeyValue, valueLength=1",
659      "Overflow when reading row length at position=8",
660      "Invalid row length in KeyValue, rowLength=1",
661      "Overflow when reading family length at position=13",
662      "Invalid family length in KeyValue, familyLength=1", "Timestamp cannot be negative, ts=-1",
663      "Invalid type in KeyValue, type=3", "Overflow when reading value part at position=25",
664      "Invalid tags length in KeyValue at position=26" };
665    byte[][] withTagsInputs = new byte[][] {
666      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
667        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x01"), // case.13
668      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
669        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x01"), // case.14
670      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
671        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x04\\x00\\x03\\x00A"), // case.15
672      // case.16
673      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
674        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0A\\x00\\x04\\x00TAG\\x00\\x04"
675        + "\\xFFT"),
676      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
677        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0C\\x00\\x04\\x00TAG\\x00\\x05"
678        + "\\xF0COME\\x00"), // case.17
679      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
680        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0C\\x00\\x04\\x00TAG\\x00\\x05"
681        + "\\xF0COME"), // case.18
682      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
683        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x00"), // case.19
684      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
685        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x1B\\x00\\x05\\x01TAG1\\x00\\x05"
686        + "\\x02TAG2\\x00\\x05\\x03TAG3\\x00\\x05\\x04TAG4"), // case.20
687    };
688    String[] withTagsOutputs = new String[] { "Overflow when reading tags length at position=26",
689      "Invalid tags length in KeyValue at position=26",
690      "Invalid tag length at position=28, tagLength=3",
691      "Invalid tag length at position=34, tagLength=4",
692      "Some redundant bytes in KeyValue's buffer, startOffset=41, endOffset=42", null, null,
693      null, };
694    assertEquals(inputs.length, outputs.length);
695    assertEquals(withTagsInputs.length, withTagsOutputs.length);
696
697    FailureCase[] cases = new FailureCase[inputs.length + withTagsInputs.length];
698    for (int i = 0; i < inputs.length; i++) {
699      cases[i] = new FailureCase(inputs[i], 0, inputs[i].length, false, outputs[i]);
700    }
701    for (int i = 0; i < withTagsInputs.length; i++) {
702      cases[inputs.length + i] =
703        new FailureCase(withTagsInputs[i], 0, withTagsInputs[i].length, true, withTagsOutputs[i]);
704    }
705
706    for (int i = 0; i < cases.length; i++) {
707      FailureCase c = cases[i];
708      ByteArrayOutputStream baos = new ByteArrayOutputStream();
709      DataOutputStream os = new DataOutputStream(baos);
710      ByteBufferUtils.putInt(os, c.length);
711      os.write(c.buf, c.offset, c.length);
712      try {
713        KeyValueUtil.createKeyValueFromInputStream(
714          new DataInputStream(new ByteArrayInputStream(baos.toByteArray())), c.withTags);
715        if (c.expectedMessage != null) {
716          fail("Should fail when parse key value from an invalid bytes for case#" + i + ". " + c);
717        }
718      } catch (IllegalArgumentException e) {
719        assertEquals(c.getExpectedMessage(), e.getMessage(), "Case#" + i + " failed," + c);
720      }
721    }
722  }
723}