001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase; 019 020import static org.junit.Assert.assertArrayEquals; 021import static org.junit.Assert.assertEquals; 022import static org.junit.Assert.assertFalse; 023import static org.junit.Assert.assertNotEquals; 024import static org.junit.Assert.assertNotNull; 025import static org.junit.Assert.assertTrue; 026import static org.junit.Assert.fail; 027 028import java.io.ByteArrayInputStream; 029import java.io.ByteArrayOutputStream; 030import java.io.DataInputStream; 031import java.io.DataOutputStream; 032import java.util.Collections; 033import java.util.Iterator; 034import java.util.List; 035import java.util.Set; 036import java.util.TreeSet; 037import org.apache.hadoop.hbase.testclassification.SmallTests; 038import org.apache.hadoop.hbase.util.ByteBufferUtils; 039import org.apache.hadoop.hbase.util.Bytes; 040import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 041import org.junit.ClassRule; 042import org.junit.Test; 043import org.junit.experimental.categories.Category; 044import org.slf4j.Logger; 045import org.slf4j.LoggerFactory; 046 047@Category(SmallTests.class) 048public class TestKeyValue { 049 @ClassRule 050 public static final HBaseClassTestRule CLASS_RULE = 051 HBaseClassTestRule.forClass(TestKeyValue.class); 052 private static final Logger LOG = LoggerFactory.getLogger(TestKeyValue.class); 053 054 @Test 055 public void testColumnCompare() { 056 final byte[] a = Bytes.toBytes("aaa"); 057 byte[] family1 = Bytes.toBytes("abc"); 058 byte[] qualifier1 = Bytes.toBytes("def"); 059 byte[] family2 = Bytes.toBytes("abcd"); 060 byte[] qualifier2 = Bytes.toBytes("ef"); 061 062 KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, KeyValue.Type.Put, a); 063 assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2)); 064 assertTrue(CellUtil.matchingColumn(aaa, family1, qualifier1)); 065 aaa = new KeyValue(a, family2, qualifier2, 0L, KeyValue.Type.Put, a); 066 assertFalse(CellUtil.matchingColumn(aaa, family1, qualifier1)); 067 assertTrue(CellUtil.matchingColumn(aaa, family2, qualifier2)); 068 byte[] nullQualifier = new byte[0]; 069 aaa = new KeyValue(a, family1, nullQualifier, 0L, KeyValue.Type.Put, a); 070 assertTrue(CellUtil.matchingColumn(aaa, family1, null)); 071 assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2)); 072 } 073 074 /** 075 * Test a corner case when the family qualifier is a prefix of the column qualifier. 076 */ 077 @Test 078 public void testColumnCompare_prefix() { 079 final byte[] a = Bytes.toBytes("aaa"); 080 byte[] family1 = Bytes.toBytes("abc"); 081 byte[] qualifier1 = Bytes.toBytes("def"); 082 byte[] family2 = Bytes.toBytes("ab"); 083 byte[] qualifier2 = Bytes.toBytes("def"); 084 085 KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, KeyValue.Type.Put, a); 086 assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2)); 087 } 088 089 @Test 090 public void testBasics() { 091 LOG.info("LOWKEY: " + KeyValue.LOWESTKEY.toString()); 092 String name = "testBasics"; 093 check(Bytes.toBytes(name), Bytes.toBytes(name), Bytes.toBytes(name), 1, Bytes.toBytes(name)); 094 // Test empty value and empty column -- both should work. (not empty fam) 095 check(Bytes.toBytes(name), Bytes.toBytes(name), null, 1, null); 096 check(HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes(name), null, 1, null); 097 // empty qual is equivalent to null qual 098 assertEquals(new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"), null, 1, (byte[]) null), 099 new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"), HConstants.EMPTY_BYTE_ARRAY, 1, 100 (byte[]) null)); 101 } 102 103 private void check(final byte[] row, final byte[] family, byte[] qualifier, final long timestamp, 104 final byte[] value) { 105 KeyValue kv = new KeyValue(row, family, qualifier, timestamp, value); 106 assertTrue( 107 Bytes.compareTo(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), row, 0, row.length) 108 == 0); 109 assertTrue(CellUtil.matchingColumn(kv, family, qualifier)); 110 // Call toString to make sure it works. 111 LOG.info(kv.toString()); 112 } 113 114 @Test 115 public void testPlainCompare() { 116 final byte[] a = Bytes.toBytes("aaa"); 117 final byte[] b = Bytes.toBytes("bbb"); 118 final byte[] fam = Bytes.toBytes("col"); 119 final byte[] qf = Bytes.toBytes("umn"); 120 KeyValue aaa = new KeyValue(a, fam, qf, a); 121 KeyValue bbb = new KeyValue(b, fam, qf, b); 122 assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) < 0); 123 assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) > 0); 124 // Compare breaks if passed same ByteBuffer as both left and right arguments. 125 assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, bbb) == 0); 126 assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0); 127 // Do compare with different timestamps. 128 aaa = new KeyValue(a, fam, qf, 1, a); 129 bbb = new KeyValue(a, fam, qf, 2, a); 130 assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) > 0); 131 assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) < 0); 132 assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0); 133 // Do compare with different types. Higher numbered types -- Delete 134 // should sort ahead of lower numbers; i.e. Put 135 aaa = new KeyValue(a, fam, qf, 1, KeyValue.Type.Delete, a); 136 bbb = new KeyValue(a, fam, qf, 1, a); 137 assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, bbb) < 0); 138 assertTrue(CellComparatorImpl.COMPARATOR.compare(bbb, aaa) > 0); 139 assertTrue(CellComparatorImpl.COMPARATOR.compare(aaa, aaa) == 0); 140 } 141 142 @Test 143 public void testMoreComparisons() { 144 long now = EnvironmentEdgeManager.currentTime(); 145 146 // Meta compares 147 KeyValue aaa = 148 new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236020145502"), now); 149 KeyValue bbb = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,,99999999999999"), now); 150 CellComparator c = MetaCellComparator.META_COMPARATOR; 151 assertTrue(c.compare(bbb, aaa) < 0); 152 153 KeyValue aaaa = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,,1236023996656"), 154 Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236024396271L, (byte[]) null); 155 assertTrue(c.compare(aaaa, bbb) < 0); 156 157 KeyValue x = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"), 158 Bytes.toBytes("info"), Bytes.toBytes(""), 9223372036854775807L, (byte[]) null); 159 KeyValue y = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"), 160 Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236034574912L, (byte[]) null); 161 assertTrue(c.compare(x, y) < 0); 162 comparisons(MetaCellComparator.META_COMPARATOR); 163 comparisons(CellComparatorImpl.COMPARATOR); 164 metacomparisons(MetaCellComparator.META_COMPARATOR); 165 } 166 167 @Test 168 public void testMetaComparatorTableKeysWithCommaOk() { 169 CellComparator c = MetaCellComparator.META_COMPARATOR; 170 long now = EnvironmentEdgeManager.currentTime(); 171 // meta keys values are not quite right. A users can enter illegal values 172 // from shell when scanning meta. 173 KeyValue a = new KeyValue(Bytes.toBytes("table,key,with,commas1,1234"), now); 174 KeyValue b = new KeyValue(Bytes.toBytes("table,key,with,commas2,0123"), now); 175 assertTrue(c.compare(a, b) < 0); 176 } 177 178 /** 179 * Tests cases where rows keys have characters below the ','. See HBASE-832 180 */ 181 @Test 182 public void testKeyValueBorderCases() { 183 // % sorts before , so if we don't do special comparator, rowB would 184 // come before rowA. 185 KeyValue rowA = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/,1234"), 186 Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[]) null); 187 KeyValue rowB = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/%20,99999"), 188 Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[]) null); 189 assertTrue(MetaCellComparator.META_COMPARATOR.compare(rowA, rowB) < 0); 190 191 rowA = new KeyValue(Bytes.toBytes("testtable,,1234"), Bytes.toBytes("fam"), Bytes.toBytes(""), 192 Long.MAX_VALUE, (byte[]) null); 193 rowB = new KeyValue(Bytes.toBytes("testtable,$www.hbase.org/,99999"), Bytes.toBytes("fam"), 194 Bytes.toBytes(""), Long.MAX_VALUE, (byte[]) null); 195 assertTrue(MetaCellComparator.META_COMPARATOR.compare(rowA, rowB) < 0); 196 } 197 198 private void metacomparisons(final CellComparatorImpl c) { 199 long now = EnvironmentEdgeManager.currentTime(); 200 assertTrue(c.compare( 201 new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now), 202 new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now)) 203 == 0); 204 KeyValue a = 205 new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now); 206 KeyValue b = 207 new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,2"), now); 208 assertTrue(c.compare(a, b) < 0); 209 assertTrue(c.compare( 210 new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,2"), now), 211 new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",a,,0,1"), now)) 212 > 0); 213 } 214 215 private void comparisons(final CellComparatorImpl c) { 216 long now = EnvironmentEdgeManager.currentTime(); 217 assertTrue(c.compare( 218 new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now), 219 new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now)) == 0); 220 assertTrue(c.compare( 221 new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now), 222 new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,2"), now)) < 0); 223 assertTrue(c.compare( 224 new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,2"), now), 225 new KeyValue(Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString() + ",,1"), now)) > 0); 226 } 227 228 @Test 229 public void testBinaryKeys() { 230 Set<KeyValue> set = new TreeSet<>(CellComparatorImpl.COMPARATOR); 231 final byte[] fam = Bytes.toBytes("col"); 232 final byte[] qf = Bytes.toBytes("umn"); 233 final byte[] nb = new byte[0]; 234 KeyValue[] keys = { new KeyValue(Bytes.toBytes("aaaaa,\u0000\u0000,2"), fam, qf, 2, nb), 235 new KeyValue(Bytes.toBytes("aaaaa,\u0001,3"), fam, qf, 3, nb), 236 new KeyValue(Bytes.toBytes("aaaaa,,1"), fam, qf, 1, nb), 237 new KeyValue(Bytes.toBytes("aaaaa,\u1000,5"), fam, qf, 5, nb), 238 new KeyValue(Bytes.toBytes("aaaaa,a,4"), fam, qf, 4, nb), 239 new KeyValue(Bytes.toBytes("a,a,0"), fam, qf, 0, nb), }; 240 // Add to set with bad comparator 241 Collections.addAll(set, keys); 242 // This will output the keys incorrectly. 243 boolean assertion = false; 244 int count = 0; 245 try { 246 for (KeyValue k : set) { 247 assertEquals(count++, k.getTimestamp()); 248 } 249 } catch (java.lang.AssertionError e) { 250 // Expected 251 assertion = true; 252 } 253 assertTrue(assertion); 254 // Make set with good comparator 255 set = new TreeSet<>(MetaCellComparator.META_COMPARATOR); 256 Collections.addAll(set, keys); 257 count = 0; 258 for (KeyValue k : set) { 259 assertEquals(count++, k.getTimestamp()); 260 } 261 } 262 263 @Test 264 public void testStackedUpKeyValue() { 265 // Test multiple KeyValues in a single blob. 266 267 // TODO actually write this test! 268 } 269 270 private final byte[] rowA = Bytes.toBytes("rowA"); 271 private final byte[] rowB = Bytes.toBytes("rowB"); 272 273 private final byte[] family = Bytes.toBytes("family"); 274 private final byte[] qualA = Bytes.toBytes("qfA"); 275 276 private void assertKVLess(CellComparator c, KeyValue less, KeyValue greater) { 277 int cmp = c.compare(less, greater); 278 assertTrue(cmp < 0); 279 cmp = c.compare(greater, less); 280 assertTrue(cmp > 0); 281 } 282 283 private void assertKVLessWithoutRow(CellComparator c, KeyValue less, KeyValue greater) { 284 int cmp = c.compare(less, greater); 285 assertTrue(cmp < 0); 286 cmp = c.compare(greater, less); 287 assertTrue(cmp > 0); 288 } 289 290 @Test 291 public void testCompareWithoutRow() { 292 final CellComparator c = CellComparatorImpl.COMPARATOR; 293 byte[] row = Bytes.toBytes("row"); 294 295 byte[] fa = Bytes.toBytes("fa"); 296 byte[] fami = Bytes.toBytes("fami"); 297 byte[] fami1 = Bytes.toBytes("fami1"); 298 299 byte[] qual0 = Bytes.toBytes(""); 300 byte[] qual1 = Bytes.toBytes("qf1"); 301 byte[] qual2 = Bytes.toBytes("qf2"); 302 long ts = 1; 303 304 // 'fa:' 305 KeyValue kv_0 = new KeyValue(row, fa, qual0, ts, KeyValue.Type.Put); 306 // 'fami:' 307 KeyValue kv0_0 = new KeyValue(row, fami, qual0, ts, KeyValue.Type.Put); 308 // 'fami:qf1' 309 KeyValue kv0_1 = new KeyValue(row, fami, qual1, ts, KeyValue.Type.Put); 310 // 'fami:qf2' 311 KeyValue kv0_2 = new KeyValue(row, fami, qual2, ts, KeyValue.Type.Put); 312 // 'fami1:' 313 KeyValue kv1_0 = new KeyValue(row, fami1, qual0, ts, KeyValue.Type.Put); 314 315 // 'fami:qf1' < 'fami:qf2' 316 assertKVLessWithoutRow(c, kv0_1, kv0_2); 317 // 'fami:qf1' < 'fami1:' 318 assertKVLessWithoutRow(c, kv0_1, kv1_0); 319 320 // Test comparison by skipping the same prefix bytes. 321 /* 322 * KeyValue Format and commonLength: 323 * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|.... 324 * ------------------|-------commonLength--------|-------------- 325 */ 326 int commonLength = KeyValue.ROW_LENGTH_SIZE + KeyValue.FAMILY_LENGTH_SIZE + row.length; 327 // 'fa:' < 'fami:'. They have commonPrefix + 2 same prefix bytes. 328 assertKVLessWithoutRow(c, kv_0, kv0_0); 329 // 'fami:' < 'fami:qf1'. They have commonPrefix + 4 same prefix bytes. 330 assertKVLessWithoutRow(c, kv0_0, kv0_1); 331 // 'fami:qf1' < 'fami1:'. They have commonPrefix + 4 same prefix bytes. 332 assertKVLessWithoutRow(c, kv0_1, kv1_0); 333 // 'fami:qf1' < 'fami:qf2'. They have commonPrefix + 6 same prefix bytes. 334 assertKVLessWithoutRow(c, kv0_1, kv0_2); 335 } 336 337 @Test 338 public void testFirstLastOnRow() { 339 final CellComparator c = CellComparatorImpl.COMPARATOR; 340 long ts = 1; 341 byte[] bufferA = new byte[128]; 342 int offsetA = 0; 343 byte[] bufferB = new byte[128]; 344 int offsetB = 7; 345 346 // These are listed in sort order (ie: every one should be less 347 // than the one on the next line). 348 final KeyValue firstOnRowA = KeyValueUtil.createFirstOnRow(rowA); 349 final KeyValue firstOnRowABufferFamQual = KeyValueUtil.createFirstOnRow(bufferA, offsetA, rowA, 350 0, rowA.length, family, 0, family.length, qualA, 0, qualA.length); 351 final KeyValue kvA_1 = new KeyValue(rowA, null, null, ts, KeyValue.Type.Put); 352 final KeyValue kvA_2 = new KeyValue(rowA, family, qualA, ts, KeyValue.Type.Put); 353 354 final KeyValue lastOnRowA = KeyValueUtil.createLastOnRow(rowA); 355 final KeyValue firstOnRowB = KeyValueUtil.createFirstOnRow(rowB); 356 final KeyValue firstOnRowBBufferFam = KeyValueUtil.createFirstOnRow(bufferB, offsetB, rowB, 0, 357 rowB.length, family, 0, family.length, null, 0, 0); 358 final KeyValue kvB = new KeyValue(rowB, family, qualA, ts, KeyValue.Type.Put); 359 360 assertKVLess(c, firstOnRowA, firstOnRowB); 361 assertKVLess(c, firstOnRowA, firstOnRowBBufferFam); 362 assertKVLess(c, firstOnRowABufferFamQual, firstOnRowB); 363 assertKVLess(c, firstOnRowA, kvA_1); 364 assertKVLess(c, firstOnRowA, kvA_2); 365 assertKVLess(c, firstOnRowABufferFamQual, kvA_2); 366 assertKVLess(c, kvA_1, kvA_2); 367 assertKVLess(c, kvA_2, firstOnRowB); 368 assertKVLess(c, kvA_1, firstOnRowB); 369 assertKVLess(c, kvA_2, firstOnRowBBufferFam); 370 assertKVLess(c, kvA_1, firstOnRowBBufferFam); 371 372 assertKVLess(c, lastOnRowA, firstOnRowB); 373 assertKVLess(c, lastOnRowA, firstOnRowBBufferFam); 374 assertKVLess(c, firstOnRowB, kvB); 375 assertKVLess(c, firstOnRowBBufferFam, kvB); 376 assertKVLess(c, lastOnRowA, kvB); 377 378 assertKVLess(c, kvA_2, lastOnRowA); 379 assertKVLess(c, kvA_1, lastOnRowA); 380 assertKVLess(c, firstOnRowA, lastOnRowA); 381 assertKVLess(c, firstOnRowABufferFamQual, lastOnRowA); 382 } 383 384 @Test 385 public void testCreateKeyOnly() { 386 long ts = 1; 387 byte[] value = Bytes.toBytes("a real value"); 388 byte[] evalue = new byte[0]; // empty value 389 390 for (byte[] val : new byte[][] { value, evalue }) { 391 for (boolean useLen : new boolean[] { false, true }) { 392 KeyValue kv1 = new KeyValue(rowA, family, qualA, ts, val); 393 KeyValue kv1ko = kv1.createKeyOnly(useLen); 394 // keys are still the same 395 assertTrue(kv1.equals(kv1ko)); 396 // but values are not 397 assertTrue(kv1ko.getValueLength() == (useLen ? Bytes.SIZEOF_INT : 0)); 398 if (useLen) { 399 assertEquals(kv1.getValueLength(), 400 Bytes.toInt(kv1ko.getValueArray(), kv1ko.getValueOffset(), kv1ko.getValueLength())); 401 } 402 } 403 } 404 } 405 406 @Test 407 public void testCreateKeyValueFromKey() { 408 KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), 409 Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("myValue")); 410 int initialPadding = 10; 411 int endingPadding = 20; 412 int keyLen = kv.getKeyLength(); 413 byte[] tmpArr = new byte[initialPadding + endingPadding + keyLen]; 414 System.arraycopy(kv.getBuffer(), kv.getKeyOffset(), tmpArr, initialPadding, keyLen); 415 KeyValue kvFromKey = KeyValueUtil.createKeyValueFromKey(tmpArr, initialPadding, keyLen); 416 assertEquals(keyLen, kvFromKey.getKeyLength()); 417 assertEquals(KeyValue.ROW_OFFSET + keyLen, kvFromKey.getBuffer().length); 418 System.err.println("kv=" + kv); 419 System.err.println("kvFromKey=" + kvFromKey); 420 assertEquals(kvFromKey.toString(), kv.toString().replaceAll("=[0-9]+", "=0")); 421 } 422 423 /** 424 * Tests that getTimestamp() does always return the proper timestamp, even after updating it. See 425 * HBASE-6265. 426 */ 427 @Test 428 public void testGetTimestamp() { 429 KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), 430 Bytes.toBytes("myQualifier"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("myValue")); 431 long time1 = kv.getTimestamp(); 432 kv.updateLatestStamp(Bytes.toBytes(12345L)); 433 long time2 = kv.getTimestamp(); 434 assertEquals(HConstants.LATEST_TIMESTAMP, time1); 435 assertEquals(12345L, time2); 436 } 437 438 @Test 439 public void testKVsWithTags() { 440 byte[] row = Bytes.toBytes("myRow"); 441 byte[] cf = Bytes.toBytes("myCF"); 442 byte[] q = Bytes.toBytes("myQualifier"); 443 byte[] value = Bytes.toBytes("myValue"); 444 byte[] metaValue1 = Bytes.toBytes("metaValue1"); 445 byte[] metaValue2 = Bytes.toBytes("metaValue2"); 446 KeyValue kv = new KeyValue(row, cf, q, HConstants.LATEST_TIMESTAMP, value, new Tag[] { 447 new ArrayBackedTag((byte) 1, metaValue1), new ArrayBackedTag((byte) 2, metaValue2) }); 448 assertTrue(kv.getTagsLength() > 0); 449 assertTrue( 450 Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), row, 0, row.length)); 451 assertTrue(Bytes.equals(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), cf, 0, 452 cf.length)); 453 assertTrue(Bytes.equals(kv.getQualifierArray(), kv.getQualifierOffset(), 454 kv.getQualifierLength(), q, 0, q.length)); 455 assertTrue(Bytes.equals(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength(), value, 0, 456 value.length)); 457 List<Tag> tags = PrivateCellUtil.getTags(kv); 458 assertNotNull(tags); 459 assertEquals(2, tags.size()); 460 boolean meta1Ok = false, meta2Ok = false; 461 for (Tag tag : tags) { 462 if (tag.getType() == (byte) 1) { 463 if (Bytes.equals(Tag.cloneValue(tag), metaValue1)) { 464 meta1Ok = true; 465 } 466 } else { 467 if (Bytes.equals(Tag.cloneValue(tag), metaValue2)) { 468 meta2Ok = true; 469 } 470 } 471 } 472 assertTrue(meta1Ok); 473 assertTrue(meta2Ok); 474 Iterator<Tag> tagItr = PrivateCellUtil.tagsIterator(kv); 475 476 assertTrue(tagItr.hasNext()); 477 Tag next = tagItr.next(); 478 assertEquals(10, next.getValueLength()); 479 assertEquals((byte) 1, next.getType()); 480 Bytes.equals(Tag.cloneValue(next), metaValue1); 481 assertTrue(tagItr.hasNext()); 482 next = tagItr.next(); 483 assertEquals(10, next.getValueLength()); 484 assertEquals((byte) 2, next.getType()); 485 Bytes.equals(Tag.cloneValue(next), metaValue2); 486 assertFalse(tagItr.hasNext()); 487 488 tagItr = PrivateCellUtil.tagsIterator(kv); 489 assertTrue(tagItr.hasNext()); 490 next = tagItr.next(); 491 assertEquals(10, next.getValueLength()); 492 assertEquals((byte) 1, next.getType()); 493 Bytes.equals(Tag.cloneValue(next), metaValue1); 494 assertTrue(tagItr.hasNext()); 495 next = tagItr.next(); 496 assertEquals(10, next.getValueLength()); 497 assertEquals((byte) 2, next.getType()); 498 Bytes.equals(Tag.cloneValue(next), metaValue2); 499 assertFalse(tagItr.hasNext()); 500 } 501 502 @Test 503 public void testMetaKeyComparator() { 504 CellComparator c = MetaCellComparator.META_COMPARATOR; 505 long now = EnvironmentEdgeManager.currentTime(); 506 507 KeyValue a = new KeyValue(Bytes.toBytes("table1"), now); 508 KeyValue b = new KeyValue(Bytes.toBytes("table2"), now); 509 assertTrue(c.compare(a, b) < 0); 510 511 a = new KeyValue(Bytes.toBytes("table1,111"), now); 512 b = new KeyValue(Bytes.toBytes("table2"), now); 513 assertTrue(c.compare(a, b) < 0); 514 515 a = new KeyValue(Bytes.toBytes("table1"), now); 516 b = new KeyValue(Bytes.toBytes("table2,111"), now); 517 assertTrue(c.compare(a, b) < 0); 518 519 a = new KeyValue(Bytes.toBytes("table,111"), now); 520 b = new KeyValue(Bytes.toBytes("table,2222"), now); 521 assertTrue(c.compare(a, b) < 0); 522 523 a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now); 524 b = new KeyValue(Bytes.toBytes("table,2222"), now); 525 assertTrue(c.compare(a, b) < 0); 526 527 a = new KeyValue(Bytes.toBytes("table,111"), now); 528 b = new KeyValue(Bytes.toBytes("table,2222.bbb"), now); 529 assertTrue(c.compare(a, b) < 0); 530 531 a = new KeyValue(Bytes.toBytes("table,,aaaa"), now); 532 b = new KeyValue(Bytes.toBytes("table,111,bbb"), now); 533 assertTrue(c.compare(a, b) < 0); 534 535 a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now); 536 b = new KeyValue(Bytes.toBytes("table,111,bbb"), now); 537 assertTrue(c.compare(a, b) < 0); 538 539 a = new KeyValue(Bytes.toBytes("table,111,xxxx"), now); 540 b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now); 541 assertTrue(c.compare(a, b) < 0); 542 543 a = new KeyValue(Bytes.toBytes("table,111,11,xxx"), now); 544 b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now); 545 assertTrue(c.compare(a, b) < 0); 546 } 547 548 @Test 549 public void testEqualsAndHashCode() { 550 KeyValue kvA1 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"), 551 Bytes.toBytes("1")); 552 KeyValue kvA2 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"), 553 Bytes.toBytes("2")); 554 // We set a different sequence id on kvA2 to demonstrate that the equals and hashCode also 555 // don't take this into account. 556 kvA2.setSequenceId(2); 557 KeyValue kvB = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualB"), 558 Bytes.toBytes("1")); 559 560 assertEquals(kvA1, kvA2); 561 assertNotEquals(kvA1, kvB); 562 assertEquals(kvA1.hashCode(), kvA2.hashCode()); 563 assertNotEquals(kvA1.hashCode(), kvB.hashCode()); 564 } 565 566 @Test 567 public void testKeyValueSerialization() throws Exception { 568 KeyValue[] keyValues = new KeyValue[] { 569 new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"), 570 Bytes.toBytes("1")), 571 new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"), 572 Bytes.toBytes("2")), 573 new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"), 574 EnvironmentEdgeManager.currentTime(), Bytes.toBytes("2"), 575 new Tag[] { new ArrayBackedTag((byte) 120, "tagA"), 576 new ArrayBackedTag((byte) 121, Bytes.toBytes("tagB")) }), 577 new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"), 578 EnvironmentEdgeManager.currentTime(), Bytes.toBytes("2"), 579 new Tag[] { new ArrayBackedTag((byte) 0, "tagA") }), 580 new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes(""), 581 Bytes.toBytes("1")) }; 582 ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); 583 for (KeyValue kv : keyValues) { 584 DataOutputStream os = new DataOutputStream(byteArrayOutputStream); 585 ByteBufferUtils.putInt(os, KeyValueUtil.getSerializedSize(kv, true)); 586 KeyValueUtil.oswrite(kv, os, true); 587 } 588 DataInputStream is = 589 new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray())); 590 for (int i = 0; i < keyValues.length; i++) { 591 LOG.info("Case#" + i + ": deserialize the kv: " + keyValues[i]); 592 KeyValue destKv = KeyValueUtil.createKeyValueFromInputStream(is, true); 593 assertEquals(keyValues[i], destKv); 594 assertArrayEquals(CellUtil.cloneValue(keyValues[i]), CellUtil.cloneValue(destKv)); 595 assertArrayEquals(PrivateCellUtil.cloneTags(keyValues[i]), PrivateCellUtil.cloneTags(destKv)); 596 } 597 } 598 599 @Test 600 public void testNullByteArrayKeyValueFailure() { 601 // can't add to testCheckKeyValueBytesFailureCase because it 602 // goes through the InputStream KeyValue API which can't produce a null buffer 603 try { 604 new KeyValue(null, 0, 0); 605 } catch (IllegalArgumentException iae) { 606 assertEquals("Invalid to have null byte array in KeyValue.", iae.getMessage()); 607 return; 608 } 609 fail("Should have thrown an IllegalArgumentException when " 610 + "creating a KeyValue with a null buffer"); 611 } 612 613 private static class FailureCase { 614 byte[] buf; 615 int offset; 616 int length; 617 boolean withTags; 618 String expectedMessage; 619 620 public FailureCase(byte[] buf, int offset, int length, boolean withTags, 621 String expectedMessage) { 622 this.buf = buf; 623 this.offset = offset; 624 this.length = length; 625 this.withTags = withTags; 626 this.expectedMessage = expectedMessage; 627 } 628 629 @Override 630 public String toString() { 631 return "FailureCaseDetails: [buf=" + Bytes.toStringBinary(buf, offset, length) + ", offset=" 632 + offset + ", " + "length=" + length + ", expectedMessage=" + expectedMessage 633 + ", withtags=" + withTags + "]"; 634 } 635 636 public String getExpectedMessage() { 637 return this.expectedMessage + KeyValueUtil.bytesToHex(buf, offset, length); 638 } 639 } 640 641 @Test 642 public void testCheckKeyValueBytesFailureCase() throws Exception { 643 byte[][] inputs = new byte[][] { HConstants.EMPTY_BYTE_ARRAY, // case.0 644 Bytes.toBytesBinary("a"), // case.1 645 Bytes.toBytesBinary("\\x00\\x00\\x00\\x01"), // case.2 646 Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00"), // case.3 647 Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01"), // case.4 648 Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00"), // case.5 649 Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x01"), // case.6 650 Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x03ROW"), // case.7 651 Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01"), // case.8 652 Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\xFF" 653 + "\\xFF\\xFF\\xFF\\xFF\\xFF\\xFF\\xFF\\x03"), // case.9 654 Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00" 655 + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x03"), // case.10 656 Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00" 657 + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04"), // case.11 658 Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00" 659 + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04VALUE"), // case.12 660 }; 661 String[] outputs = new String[] { "Overflow when reading key length at position=0", 662 "Overflow when reading key length at position=0", 663 "Invalid key length in KeyValue. keyLength=1", 664 "Overflow when reading value length at position=4", 665 "Invalid value length in KeyValue, valueLength=1", 666 "Overflow when reading row length at position=8", 667 "Invalid row length in KeyValue, rowLength=1", 668 "Overflow when reading family length at position=13", 669 "Invalid family length in KeyValue, familyLength=1", "Timestamp cannot be negative, ts=-1", 670 "Invalid type in KeyValue, type=3", "Overflow when reading value part at position=25", 671 "Invalid tags length in KeyValue at position=26" }; 672 byte[][] withTagsInputs = new byte[][] { 673 Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00" 674 + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x01"), // case.13 675 Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00" 676 + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x01"), // case.14 677 Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00" 678 + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x04\\x00\\x03\\x00A"), // case.15 679 // case.16 680 Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00" 681 + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0A\\x00\\x04\\x00TAG\\x00\\x04" 682 + "\\xFFT"), 683 Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00" 684 + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0C\\x00\\x04\\x00TAG\\x00\\x05" 685 + "\\xF0COME\\x00"), // case.17 686 Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00" 687 + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0C\\x00\\x04\\x00TAG\\x00\\x05" 688 + "\\xF0COME"), // case.18 689 Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00" 690 + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x00"), // case.19 691 Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00" 692 + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x1B\\x00\\x05\\x01TAG1\\x00\\x05" 693 + "\\x02TAG2\\x00\\x05\\x03TAG3\\x00\\x05\\x04TAG4"), // case.20 694 }; 695 String[] withTagsOutputs = new String[] { "Overflow when reading tags length at position=26", 696 "Invalid tags length in KeyValue at position=26", 697 "Invalid tag length at position=28, tagLength=3", 698 "Invalid tag length at position=34, tagLength=4", 699 "Some redundant bytes in KeyValue's buffer, startOffset=41, endOffset=42", null, null, 700 null, }; 701 assertEquals(inputs.length, outputs.length); 702 assertEquals(withTagsInputs.length, withTagsOutputs.length); 703 704 FailureCase[] cases = new FailureCase[inputs.length + withTagsInputs.length]; 705 for (int i = 0; i < inputs.length; i++) { 706 cases[i] = new FailureCase(inputs[i], 0, inputs[i].length, false, outputs[i]); 707 } 708 for (int i = 0; i < withTagsInputs.length; i++) { 709 cases[inputs.length + i] = 710 new FailureCase(withTagsInputs[i], 0, withTagsInputs[i].length, true, withTagsOutputs[i]); 711 } 712 713 for (int i = 0; i < cases.length; i++) { 714 FailureCase c = cases[i]; 715 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 716 DataOutputStream os = new DataOutputStream(baos); 717 ByteBufferUtils.putInt(os, c.length); 718 os.write(c.buf, c.offset, c.length); 719 try { 720 KeyValueUtil.createKeyValueFromInputStream( 721 new DataInputStream(new ByteArrayInputStream(baos.toByteArray())), c.withTags); 722 if (c.expectedMessage != null) { 723 fail("Should fail when parse key value from an invalid bytes for case#" + i + ". " + c); 724 } 725 } catch (IllegalArgumentException e) { 726 assertEquals("Case#" + i + " failed," + c, c.getExpectedMessage(), e.getMessage()); 727 } 728 } 729 } 730}