Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.Assert.assertArrayEquals;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertNotEquals;
024import static org.junit.Assert.assertNotNull;
025import static org.junit.Assert.assertTrue;
026import static org.junit.Assert.fail;
027
028import java.io.ByteArrayInputStream;
029import java.io.ByteArrayOutputStream;
030import java.io.DataInputStream;
031import java.io.DataOutputStream;
032import java.util.Collections;
033import java.util.Iterator;
034import java.util.List;
035import java.util.Set;
036import java.util.TreeSet;
037import org.apache.hadoop.hbase.testclassification.SmallTests;
038import org.apache.hadoop.hbase.util.ByteBufferUtils;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
041import org.junit.ClassRule;
042import org.junit.Test;
043import org.junit.experimental.categories.Category;
044import org.slf4j.Logger;
045import org.slf4j.LoggerFactory;
046
047@Category(SmallTests.class)
048public class TestKeyValue {
049  @ClassRule
050  public static final HBaseClassTestRule CLASS_RULE =
051    HBaseClassTestRule.forClass(TestKeyValue.class);
052  private static final Logger LOG = LoggerFactory.getLogger(TestKeyValue.class);
053
054  @Test
055  public void testColumnCompare() {
056    final byte[] a = Bytes.toBytes("aaa");
057    byte[] family1 = Bytes.toBytes("abc");
058    byte[] qualifier1 = Bytes.toBytes("def");
059    byte[] family2 = Bytes.toBytes("abcd");
060    byte[] qualifier2 = Bytes.toBytes("ef");
061
062    KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, KeyValue.Type.Put, a);
063    assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
064    assertTrue(CellUtil.matchingColumn(aaa, family1, qualifier1));
065    aaa = new KeyValue(a, family2, qualifier2, 0L, KeyValue.Type.Put, a);
066    assertFalse(CellUtil.matchingColumn(aaa, family1, qualifier1));
067    assertTrue(CellUtil.matchingColumn(aaa, family2, qualifier2));
068    byte[] nullQualifier = new byte[0];
069    aaa = new KeyValue(a, family1, nullQualifier, 0L, KeyValue.Type.Put, a);
070    assertTrue(CellUtil.matchingColumn(aaa, family1, null));
071    assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
072  }
073
074  /**
075   * Test a corner case when the family qualifier is a prefix of the column qualifier.
076   */
077  @Test
078  public void testColumnCompare_prefix() {
079    final byte[] a = Bytes.toBytes("aaa");
080    byte[] family1 = Bytes.toBytes("abc");
081    byte[] qualifier1 = Bytes.toBytes("def");
082    byte[] family2 = Bytes.toBytes("ab");
083    byte[] qualifier2 = Bytes.toBytes("def");
084
085    KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, KeyValue.Type.Put, a);
086    assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
087  }
088
089  @Test
090  public void testBasics() {
091    LOG.info("LOWKEY: " + KeyValue.LOWESTKEY.toString());
092    String name = "testBasics";
093    check(Bytes.toBytes(name), Bytes.toBytes(name), Bytes.toBytes(name), 1, Bytes.toBytes(name));
094    // Test empty value and empty column -- both should work. (not empty fam)
095    check(Bytes.toBytes(name), Bytes.toBytes(name), null, 1, null);
096    check(HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes(name), null, 1, null);
097    // empty qual is equivalent to null qual
098    assertEquals(new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"), null, 1, (byte[]) null),
099      new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"), HConstants.EMPTY_BYTE_ARRAY, 1,
100        (byte[]) null));
101  }
102
103  private void check(final byte[] row, final byte[] family, byte[] qualifier, final long timestamp,
104    final byte[] value) {
105    KeyValue kv = new KeyValue(row, family, qualifier, timestamp, value);
106    assertTrue(
107      Bytes.compareTo(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), row, 0, row.length)
108          == 0);
109    assertTrue(CellUtil.matchingColumn(kv, family, qualifier));
110    // Call toString to make sure it works.
111    LOG.info(kv.toString());
112  }
113
114  @Test
115  public void testPlainCompare() {
116    final byte[] a = Bytes.toBytes("aaa");
117    final byte[] b = Bytes.toBytes("bbb");
118    final byte[] fam = Bytes.toBytes("col");
119    final byte[] qf = Bytes.toBytes("umn");
120    KeyValue aaa = new KeyValue(a, fam, qf, a);
121    KeyValue bbb = new KeyValue(b, fam, qf, b);
122    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) < 0);
123    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) > 0);
124    // Compare breaks if passed same ByteBuffer as both left and right arguments.
125    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, bbb) == 0);
126    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0);
127    // Do compare with different timestamps.
128    aaa = new KeyValue(a, fam, qf, 1, a);
129    bbb = new KeyValue(a, fam, qf, 2, a);
130    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) > 0);
131    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) < 0);
132    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0);
133    // Do compare with different types. Higher numbered types -- Delete
134    // should sort ahead of lower numbers; i.e. Put
135    aaa = new KeyValue(a, fam, qf, 1, KeyValue.Type.Delete, a);
136    bbb = new KeyValue(a, fam, qf, 1, a);
137    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) < 0);
138    assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) > 0);
139    assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0);
140  }
141
142  @Test
143  public void testMoreComparisons() {
144    long now = EnvironmentEdgeManager.currentTime();
145
146    // Meta compares
147    KeyValue aaa =
148      new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236020145502"), now);
149    KeyValue bbb = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,,99999999999999"), now);
150    CellComparator c = MetaCellComparator.META_COMPARATOR;
151    assertTrue(c.compare(bbb, aaa) < 0);
152
153    KeyValue aaaa = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,,1236023996656"),
154      Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236024396271L, (byte[]) null);
155    assertTrue(c.compare(aaaa, bbb) < 0);
156
157    KeyValue x = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
158      Bytes.toBytes("info"), Bytes.toBytes(""), 9223372036854775807L, (byte[]) null);
159    KeyValue y = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
160      Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236034574912L, (byte[]) null);
161    assertTrue(c.compare(x, y) < 0);
162    comparisons(MetaCellComparator.META_COMPARATOR);
163    comparisons(CellComparatorImpl.COMPARATOR);
164    metacomparisons(MetaCellComparator.META_COMPARATOR);
165  }
166
167  @Test
168  public void testMetaComparatorTableKeysWithCommaOk() {
169    CellComparator c = MetaCellComparator.META_COMPARATOR;
170    long now = EnvironmentEdgeManager.currentTime();
171    // meta keys values are not quite right. A users can enter illegal values
172    // from shell when scanning meta.
173    KeyValue a = new KeyValue(Bytes.toBytes("table,key,with,commas1,1234"), now);
174    KeyValue b = new KeyValue(Bytes.toBytes("table,key,with,commas2,0123"), now);
175    assertTrue(c.compare(a, b) < 0);
176  }
177
178  /**
179   * Tests cases where rows keys have characters below the ','. See HBASE-832
180   */
181  @Test
182  public void testKeyValueBorderCases() {
183    // % sorts before , so if we don't do special comparator, rowB would
184    // come before rowA.
185    KeyValue rowA = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/,1234"),
186      Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[]) null);
187    KeyValue rowB = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/%20,99999"),
188      Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[]) null);
189    assertTrue(MetaCellComparator.META_COMPARATOR.compare(rowA, rowB) < 0);
190
191    rowA = new KeyValue(Bytes.toBytes("testtable,,1234"), Bytes.toBytes("fam"), Bytes.toBytes(""),
192      Long.MAX_VALUE, (byte[]) null);
193    rowB = new KeyValue(Bytes.toBytes("testtable,$www.hbase.org/,99999"), Bytes.toBytes("fam"),
194      Bytes.toBytes(""), Long.MAX_VALUE, (byte[]) null);
195    assertTrue(MetaCellComparator.META_COMPARATOR.compare(rowA, rowB) < 0);
196  }
197
198  private void metacomparisons(final CellComparatorImpl c) {
199    long now = EnvironmentEdgeManager.currentTime();
200    assertTrue(c.compare(
201      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now),
202      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now))
203        == 0);
204    KeyValue a =
205      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now);
206    KeyValue b =
207      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,2"), now);
208    assertTrue(c.compare(a, b) < 0);
209    assertTrue(c.compare(
210      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,2"), now),
211      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now))
212        > 0);
213  }
214
215  private void comparisons(final CellComparatorImpl c) {
216    long now = EnvironmentEdgeManager.currentTime();
217    assertTrue(c.compare(
218      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now),
219      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now)) == 0);
220    assertTrue(c.compare(
221      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now),
222      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,2"), now)) < 0);
223    assertTrue(c.compare(
224      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,2"), now),
225      new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now)) > 0);
226  }
227
228  @Test
229  public void testBinaryKeys() {
230    Set<KeyValue> set = new TreeSet<>(CellComparatorImpl.COMPARATOR);
231    final byte[] fam = Bytes.toBytes("col");
232    final byte[] qf = Bytes.toBytes("umn");
233    final byte[] nb = new byte[0];
234    KeyValue[] keys = { new KeyValue(Bytes.toBytes("aaaaa,\u0000\u0000,2"), fam, qf, 2, nb),
235      new KeyValue(Bytes.toBytes("aaaaa,\u0001,3"), fam, qf, 3, nb),
236      new KeyValue(Bytes.toBytes("aaaaa,,1"), fam, qf, 1, nb),
237      new KeyValue(Bytes.toBytes("aaaaa,\u1000,5"), fam, qf, 5, nb),
238      new KeyValue(Bytes.toBytes("aaaaa,a,4"), fam, qf, 4, nb),
239      new KeyValue(Bytes.toBytes("a,a,0"), fam, qf, 0, nb), };
240    // Add to set with bad comparator
241    Collections.addAll(set, keys);
242    // This will output the keys incorrectly.
243    boolean assertion = false;
244    int count = 0;
245    for (KeyValue k : set) {
246      if (count++ != k.getTimestamp()) {
247        assertion = true;
248      }
249    }
250    assertTrue(assertion);
251    // Make set with good comparator
252    set = new TreeSet<>(MetaCellComparator.META_COMPARATOR);
253    Collections.addAll(set, keys);
254    count = 0;
255    for (KeyValue k : set) {
256      assertEquals(count++, k.getTimestamp());
257    }
258  }
259
260  @Test
261  public void testStackedUpKeyValue() {
262    // Test multiple KeyValues in a single blob.
263
264    // TODO actually write this test!
265  }
266
267  private final byte[] rowA = Bytes.toBytes("rowA");
268  private final byte[] rowB = Bytes.toBytes("rowB");
269
270  private final byte[] family = Bytes.toBytes("family");
271  private final byte[] qualA = Bytes.toBytes("qfA");
272
273  private void assertKVLess(CellComparator c, KeyValue less, KeyValue greater) {
274    int cmp = c.compare(less, greater);
275    assertTrue(cmp < 0);
276    cmp = c.compare(greater, less);
277    assertTrue(cmp > 0);
278  }
279
280  private void assertKVLessWithoutRow(CellComparator c, KeyValue less, KeyValue greater) {
281    int cmp = c.compare(less, greater);
282    assertTrue(cmp < 0);
283    cmp = c.compare(greater, less);
284    assertTrue(cmp > 0);
285  }
286
287  @Test
288  public void testCompareWithoutRow() {
289    final CellComparator c = CellComparatorImpl.COMPARATOR;
290    byte[] row = Bytes.toBytes("row");
291
292    byte[] fa = Bytes.toBytes("fa");
293    byte[] fami = Bytes.toBytes("fami");
294    byte[] fami1 = Bytes.toBytes("fami1");
295
296    byte[] qual0 = Bytes.toBytes("");
297    byte[] qual1 = Bytes.toBytes("qf1");
298    byte[] qual2 = Bytes.toBytes("qf2");
299    long ts = 1;
300
301    // 'fa:'
302    KeyValue kv_0 = new KeyValue(row, fa, qual0, ts, KeyValue.Type.Put);
303    // 'fami:'
304    KeyValue kv0_0 = new KeyValue(row, fami, qual0, ts, KeyValue.Type.Put);
305    // 'fami:qf1'
306    KeyValue kv0_1 = new KeyValue(row, fami, qual1, ts, KeyValue.Type.Put);
307    // 'fami:qf2'
308    KeyValue kv0_2 = new KeyValue(row, fami, qual2, ts, KeyValue.Type.Put);
309    // 'fami1:'
310    KeyValue kv1_0 = new KeyValue(row, fami1, qual0, ts, KeyValue.Type.Put);
311
312    // 'fami:qf1' < 'fami:qf2'
313    assertKVLessWithoutRow(c, kv0_1, kv0_2);
314    // 'fami:qf1' < 'fami1:'
315    assertKVLessWithoutRow(c, kv0_1, kv1_0);
316
317    // Test comparison by skipping the same prefix bytes.
318    /*
319     * KeyValue Format and commonLength:
320     * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|....
321     * ------------------|-------commonLength--------|--------------
322     */
323    // 'fa:' < 'fami:'. They have commonPrefix + 2 same prefix bytes.
324    assertKVLessWithoutRow(c, kv_0, kv0_0);
325    // 'fami:' < 'fami:qf1'. They have commonPrefix + 4 same prefix bytes.
326    assertKVLessWithoutRow(c, kv0_0, kv0_1);
327    // 'fami:qf1' < 'fami1:'. They have commonPrefix + 4 same prefix bytes.
328    assertKVLessWithoutRow(c, kv0_1, kv1_0);
329    // 'fami:qf1' < 'fami:qf2'. They have commonPrefix + 6 same prefix bytes.
330    assertKVLessWithoutRow(c, kv0_1, kv0_2);
331  }
332
333  @Test
334  public void testFirstLastOnRow() {
335    final CellComparator c = CellComparatorImpl.COMPARATOR;
336    long ts = 1;
337    byte[] bufferA = new byte[128];
338    int offsetA = 0;
339    byte[] bufferB = new byte[128];
340    int offsetB = 7;
341
342    // These are listed in sort order (ie: every one should be less
343    // than the one on the next line).
344    final KeyValue firstOnRowA = KeyValueUtil.createFirstOnRow(rowA);
345    final KeyValue firstOnRowABufferFamQual = KeyValueUtil.createFirstOnRow(bufferA, offsetA, rowA,
346      0, rowA.length, family, 0, family.length, qualA, 0, qualA.length);
347    final KeyValue kvA_1 = new KeyValue(rowA, null, null, ts, KeyValue.Type.Put);
348    final KeyValue kvA_2 = new KeyValue(rowA, family, qualA, ts, KeyValue.Type.Put);
349
350    final KeyValue lastOnRowA = KeyValueUtil.createLastOnRow(rowA);
351    final KeyValue firstOnRowB = KeyValueUtil.createFirstOnRow(rowB);
352    final KeyValue firstOnRowBBufferFam = KeyValueUtil.createFirstOnRow(bufferB, offsetB, rowB, 0,
353      rowB.length, family, 0, family.length, null, 0, 0);
354    final KeyValue kvB = new KeyValue(rowB, family, qualA, ts, KeyValue.Type.Put);
355
356    assertKVLess(c, firstOnRowA, firstOnRowB);
357    assertKVLess(c, firstOnRowA, firstOnRowBBufferFam);
358    assertKVLess(c, firstOnRowABufferFamQual, firstOnRowB);
359    assertKVLess(c, firstOnRowA, kvA_1);
360    assertKVLess(c, firstOnRowA, kvA_2);
361    assertKVLess(c, firstOnRowABufferFamQual, kvA_2);
362    assertKVLess(c, kvA_1, kvA_2);
363    assertKVLess(c, kvA_2, firstOnRowB);
364    assertKVLess(c, kvA_1, firstOnRowB);
365    assertKVLess(c, kvA_2, firstOnRowBBufferFam);
366    assertKVLess(c, kvA_1, firstOnRowBBufferFam);
367
368    assertKVLess(c, lastOnRowA, firstOnRowB);
369    assertKVLess(c, lastOnRowA, firstOnRowBBufferFam);
370    assertKVLess(c, firstOnRowB, kvB);
371    assertKVLess(c, firstOnRowBBufferFam, kvB);
372    assertKVLess(c, lastOnRowA, kvB);
373
374    assertKVLess(c, kvA_2, lastOnRowA);
375    assertKVLess(c, kvA_1, lastOnRowA);
376    assertKVLess(c, firstOnRowA, lastOnRowA);
377    assertKVLess(c, firstOnRowABufferFamQual, lastOnRowA);
378  }
379
380  @Test
381  public void testCreateKeyOnly() {
382    long ts = 1;
383    byte[] value = Bytes.toBytes("a real value");
384    byte[] evalue = new byte[0]; // empty value
385
386    for (byte[] val : new byte[][] { value, evalue }) {
387      for (boolean useLen : new boolean[] { false, true }) {
388        KeyValue kv1 = new KeyValue(rowA, family, qualA, ts, val);
389        KeyValue kv1ko = kv1.createKeyOnly(useLen);
390        // keys are still the same
391        assertTrue(kv1.equals(kv1ko));
392        // but values are not
393        assertTrue(kv1ko.getValueLength() == (useLen ? Bytes.SIZEOF_INT : 0));
394        if (useLen) {
395          assertEquals(kv1.getValueLength(),
396            Bytes.toInt(kv1ko.getValueArray(), kv1ko.getValueOffset(), kv1ko.getValueLength()));
397        }
398      }
399    }
400  }
401
402  @Test
403  public void testCreateKeyValueFromKey() {
404    KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
405      Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("myValue"));
406    int initialPadding = 10;
407    int endingPadding = 20;
408    int keyLen = kv.getKeyLength();
409    byte[] tmpArr = new byte[initialPadding + endingPadding + keyLen];
410    System.arraycopy(kv.getBuffer(), kv.getKeyOffset(), tmpArr, initialPadding, keyLen);
411    KeyValue kvFromKey = KeyValueUtil.createKeyValueFromKey(tmpArr, initialPadding, keyLen);
412    assertEquals(keyLen, kvFromKey.getKeyLength());
413    assertEquals(KeyValue.ROW_OFFSET + keyLen, kvFromKey.getBuffer().length);
414    System.err.println("kv=" + kv);
415    System.err.println("kvFromKey=" + kvFromKey);
416    assertEquals(kvFromKey.toString(), kv.toString().replaceAll("=[0-9]+", "=0"));
417  }
418
419  /**
420   * Tests that getTimestamp() does always return the proper timestamp, even after updating it. See
421   * HBASE-6265.
422   */
423  @Test
424  public void testGetTimestamp() {
425    KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
426      Bytes.toBytes("myQualifier"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("myValue"));
427    long time1 = kv.getTimestamp();
428    kv.updateLatestStamp(Bytes.toBytes(12345L));
429    long time2 = kv.getTimestamp();
430    assertEquals(HConstants.LATEST_TIMESTAMP, time1);
431    assertEquals(12345L, time2);
432  }
433
434  @Test
435  public void testKVsWithTags() {
436    byte[] row = Bytes.toBytes("myRow");
437    byte[] cf = Bytes.toBytes("myCF");
438    byte[] q = Bytes.toBytes("myQualifier");
439    byte[] value = Bytes.toBytes("myValue");
440    byte[] metaValue1 = Bytes.toBytes("metaValue1");
441    byte[] metaValue2 = Bytes.toBytes("metaValue2");
442    KeyValue kv = new KeyValue(row, cf, q, HConstants.LATEST_TIMESTAMP, value, new Tag[] {
443      new ArrayBackedTag((byte) 1, metaValue1), new ArrayBackedTag((byte) 2, metaValue2) });
444    assertTrue(kv.getTagsLength() > 0);
445    assertTrue(
446      Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), row, 0, row.length));
447    assertTrue(Bytes.equals(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), cf, 0,
448      cf.length));
449    assertTrue(Bytes.equals(kv.getQualifierArray(), kv.getQualifierOffset(),
450      kv.getQualifierLength(), q, 0, q.length));
451    assertTrue(Bytes.equals(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength(), value, 0,
452      value.length));
453    List<Tag> tags = PrivateCellUtil.getTags(kv);
454    assertNotNull(tags);
455    assertEquals(2, tags.size());
456    boolean meta1Ok = false, meta2Ok = false;
457    for (Tag tag : tags) {
458      if (tag.getType() == (byte) 1) {
459        if (Bytes.equals(Tag.cloneValue(tag), metaValue1)) {
460          meta1Ok = true;
461        }
462      } else {
463        if (Bytes.equals(Tag.cloneValue(tag), metaValue2)) {
464          meta2Ok = true;
465        }
466      }
467    }
468    assertTrue(meta1Ok);
469    assertTrue(meta2Ok);
470    Iterator<Tag> tagItr = PrivateCellUtil.tagsIterator(kv);
471
472    assertTrue(tagItr.hasNext());
473    Tag next = tagItr.next();
474    assertEquals(10, next.getValueLength());
475    assertEquals((byte) 1, next.getType());
476    Bytes.equals(Tag.cloneValue(next), metaValue1);
477    assertTrue(tagItr.hasNext());
478    next = tagItr.next();
479    assertEquals(10, next.getValueLength());
480    assertEquals((byte) 2, next.getType());
481    Bytes.equals(Tag.cloneValue(next), metaValue2);
482    assertFalse(tagItr.hasNext());
483
484    tagItr = PrivateCellUtil.tagsIterator(kv);
485    assertTrue(tagItr.hasNext());
486    next = tagItr.next();
487    assertEquals(10, next.getValueLength());
488    assertEquals((byte) 1, next.getType());
489    Bytes.equals(Tag.cloneValue(next), metaValue1);
490    assertTrue(tagItr.hasNext());
491    next = tagItr.next();
492    assertEquals(10, next.getValueLength());
493    assertEquals((byte) 2, next.getType());
494    Bytes.equals(Tag.cloneValue(next), metaValue2);
495    assertFalse(tagItr.hasNext());
496  }
497
498  @Test
499  public void testMetaKeyComparator() {
500    CellComparator c = MetaCellComparator.META_COMPARATOR;
501    long now = EnvironmentEdgeManager.currentTime();
502
503    KeyValue a = new KeyValue(Bytes.toBytes("table1"), now);
504    KeyValue b = new KeyValue(Bytes.toBytes("table2"), now);
505    assertTrue(c.compare(a, b) < 0);
506
507    a = new KeyValue(Bytes.toBytes("table1,111"), now);
508    b = new KeyValue(Bytes.toBytes("table2"), now);
509    assertTrue(c.compare(a, b) < 0);
510
511    a = new KeyValue(Bytes.toBytes("table1"), now);
512    b = new KeyValue(Bytes.toBytes("table2,111"), now);
513    assertTrue(c.compare(a, b) < 0);
514
515    a = new KeyValue(Bytes.toBytes("table,111"), now);
516    b = new KeyValue(Bytes.toBytes("table,2222"), now);
517    assertTrue(c.compare(a, b) < 0);
518
519    a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
520    b = new KeyValue(Bytes.toBytes("table,2222"), now);
521    assertTrue(c.compare(a, b) < 0);
522
523    a = new KeyValue(Bytes.toBytes("table,111"), now);
524    b = new KeyValue(Bytes.toBytes("table,2222.bbb"), now);
525    assertTrue(c.compare(a, b) < 0);
526
527    a = new KeyValue(Bytes.toBytes("table,,aaaa"), now);
528    b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
529    assertTrue(c.compare(a, b) < 0);
530
531    a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
532    b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
533    assertTrue(c.compare(a, b) < 0);
534
535    a = new KeyValue(Bytes.toBytes("table,111,xxxx"), now);
536    b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
537    assertTrue(c.compare(a, b) < 0);
538
539    a = new KeyValue(Bytes.toBytes("table,111,11,xxx"), now);
540    b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
541    assertTrue(c.compare(a, b) < 0);
542  }
543
544  @Test
545  public void testEqualsAndHashCode() {
546    KeyValue kvA1 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
547      Bytes.toBytes("1"));
548    KeyValue kvA2 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
549      Bytes.toBytes("2"));
550    // We set a different sequence id on kvA2 to demonstrate that the equals and hashCode also
551    // don't take this into account.
552    kvA2.setSequenceId(2);
553    KeyValue kvB = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualB"),
554      Bytes.toBytes("1"));
555
556    assertEquals(kvA1, kvA2);
557    assertNotEquals(kvA1, kvB);
558    assertEquals(kvA1.hashCode(), kvA2.hashCode());
559    assertNotEquals(kvA1.hashCode(), kvB.hashCode());
560  }
561
562  @Test
563  public void testKeyValueSerialization() throws Exception {
564    KeyValue[] keyValues = new KeyValue[] {
565      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
566        Bytes.toBytes("1")),
567      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
568        Bytes.toBytes("2")),
569      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
570        EnvironmentEdgeManager.currentTime(), Bytes.toBytes("2"),
571        new Tag[] { new ArrayBackedTag((byte) 120, "tagA"),
572          new ArrayBackedTag((byte) 121, Bytes.toBytes("tagB")) }),
573      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
574        EnvironmentEdgeManager.currentTime(), Bytes.toBytes("2"),
575        new Tag[] { new ArrayBackedTag((byte) 0, "tagA") }),
576      new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes(""),
577        Bytes.toBytes("1")) };
578    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
579    for (KeyValue kv : keyValues) {
580      DataOutputStream os = new DataOutputStream(byteArrayOutputStream);
581      ByteBufferUtils.putInt(os, KeyValueUtil.getSerializedSize(kv, true));
582      KeyValueUtil.oswrite(kv, os, true);
583    }
584    DataInputStream is =
585      new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()));
586    for (int i = 0; i < keyValues.length; i++) {
587      LOG.info("Case#" + i + ": deserialize the kv: " + keyValues[i]);
588      KeyValue destKv = KeyValueUtil.createKeyValueFromInputStream(is, true);
589      assertEquals(keyValues[i], destKv);
590      assertArrayEquals(CellUtil.cloneValue(keyValues[i]), CellUtil.cloneValue(destKv));
591      assertArrayEquals(PrivateCellUtil.cloneTags(keyValues[i]), PrivateCellUtil.cloneTags(destKv));
592    }
593  }
594
595  @Test
596  public void testNullByteArrayKeyValueFailure() {
597    // can't add to testCheckKeyValueBytesFailureCase because it
598    // goes through the InputStream KeyValue API which can't produce a null buffer
599    try {
600      new KeyValue(null, 0, 0);
601    } catch (IllegalArgumentException iae) {
602      assertEquals("Invalid to have null byte array in KeyValue.", iae.getMessage());
603      return;
604    }
605    fail("Should have thrown an IllegalArgumentException when "
606      + "creating a KeyValue with a null buffer");
607  }
608
609  private static class FailureCase {
610    byte[] buf;
611    int offset;
612    int length;
613    boolean withTags;
614    String expectedMessage;
615
616    public FailureCase(byte[] buf, int offset, int length, boolean withTags,
617      String expectedMessage) {
618      this.buf = buf;
619      this.offset = offset;
620      this.length = length;
621      this.withTags = withTags;
622      this.expectedMessage = expectedMessage;
623    }
624
625    @Override
626    public String toString() {
627      return "FailureCaseDetails: [buf=" + Bytes.toStringBinary(buf, offset, length) + ", offset="
628        + offset + ", " + "length=" + length + ", expectedMessage=" + expectedMessage
629        + ", withtags=" + withTags + "]";
630    }
631
632    public String getExpectedMessage() {
633      return this.expectedMessage + KeyValueUtil.bytesToHex(buf, offset, length);
634    }
635  }
636
637  @Test
638  public void testCheckKeyValueBytesFailureCase() throws Exception {
639    byte[][] inputs = new byte[][] { HConstants.EMPTY_BYTE_ARRAY, // case.0
640      Bytes.toBytesBinary("a"), // case.1
641      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01"), // case.2
642      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00"), // case.3
643      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01"), // case.4
644      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00"), // case.5
645      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x01"), // case.6
646      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x03ROW"), // case.7
647      Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01"), // case.8
648      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\xFF"
649        + "\\xFF\\xFF\\xFF\\xFF\\xFF\\xFF\\xFF\\x03"), // case.9
650      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
651        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x03"), // case.10
652      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
653        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04"), // case.11
654      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
655        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04VALUE"), // case.12
656    };
657    String[] outputs = new String[] { "Overflow when reading key length at position=0",
658      "Overflow when reading key length at position=0",
659      "Invalid key length in KeyValue. keyLength=1",
660      "Overflow when reading value length at position=4",
661      "Invalid value length in KeyValue, valueLength=1",
662      "Overflow when reading row length at position=8",
663      "Invalid row length in KeyValue, rowLength=1",
664      "Overflow when reading family length at position=13",
665      "Invalid family length in KeyValue, familyLength=1", "Timestamp cannot be negative, ts=-1",
666      "Invalid type in KeyValue, type=3", "Overflow when reading value part at position=25",
667      "Invalid tags length in KeyValue at position=26" };
668    byte[][] withTagsInputs = new byte[][] {
669      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
670        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x01"), // case.13
671      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
672        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x01"), // case.14
673      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
674        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x04\\x00\\x03\\x00A"), // case.15
675      // case.16
676      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
677        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0A\\x00\\x04\\x00TAG\\x00\\x04"
678        + "\\xFFT"),
679      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
680        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0C\\x00\\x04\\x00TAG\\x00\\x05"
681        + "\\xF0COME\\x00"), // case.17
682      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
683        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0C\\x00\\x04\\x00TAG\\x00\\x05"
684        + "\\xF0COME"), // case.18
685      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
686        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x00"), // case.19
687      Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
688        + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x1B\\x00\\x05\\x01TAG1\\x00\\x05"
689        + "\\x02TAG2\\x00\\x05\\x03TAG3\\x00\\x05\\x04TAG4"), // case.20
690    };
691    String[] withTagsOutputs = new String[] { "Overflow when reading tags length at position=26",
692      "Invalid tags length in KeyValue at position=26",
693      "Invalid tag length at position=28, tagLength=3",
694      "Invalid tag length at position=34, tagLength=4",
695      "Some redundant bytes in KeyValue's buffer, startOffset=41, endOffset=42", null, null,
696      null, };
697    assertEquals(inputs.length, outputs.length);
698    assertEquals(withTagsInputs.length, withTagsOutputs.length);
699
700    FailureCase[] cases = new FailureCase[inputs.length + withTagsInputs.length];
701    for (int i = 0; i < inputs.length; i++) {
702      cases[i] = new FailureCase(inputs[i], 0, inputs[i].length, false, outputs[i]);
703    }
704    for (int i = 0; i < withTagsInputs.length; i++) {
705      cases[inputs.length + i] =
706        new FailureCase(withTagsInputs[i], 0, withTagsInputs[i].length, true, withTagsOutputs[i]);
707    }
708
709    for (int i = 0; i < cases.length; i++) {
710      FailureCase c = cases[i];
711      ByteArrayOutputStream baos = new ByteArrayOutputStream();
712      DataOutputStream os = new DataOutputStream(baos);
713      ByteBufferUtils.putInt(os, c.length);
714      os.write(c.buf, c.offset, c.length);
715      try {
716        KeyValueUtil.createKeyValueFromInputStream(
717          new DataInputStream(new ByteArrayInputStream(baos.toByteArray())), c.withTags);
718        if (c.expectedMessage != null) {
719          fail("Should fail when parse kv from an invalid bytes, case#" + i + ". " + c);
720        }
721      } catch (IllegalArgumentException e) {
722        assertEquals("Case#" + i + " failed," + c, c.getExpectedMessage(), e.getMessage());
723      }
724    }
725  }
726}