1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import java.io.DataInput;
22 import java.io.EOFException;
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.io.OutputStream;
26 import java.nio.ByteBuffer;
27 import java.util.ArrayList;
28 import java.util.List;
29
30 import org.apache.hadoop.hbase.classification.InterfaceAudience;
31 import org.apache.hadoop.hbase.KeyValue.Type;
32 import org.apache.hadoop.hbase.io.util.StreamUtils;
33 import org.apache.hadoop.hbase.util.ByteBufferUtils;
34 import org.apache.hadoop.hbase.util.Bytes;
35 import org.apache.hadoop.hbase.util.IterableUtils;
36 import org.apache.hadoop.hbase.util.SimpleMutableByteRange;
37 import org.apache.hadoop.io.IOUtils;
38 import org.apache.hadoop.io.WritableUtils;
39
40 import com.google.common.base.Function;
41 import com.google.common.collect.Lists;
42
43
44
45
46 @InterfaceAudience.Private
47 public class KeyValueUtil {
48
49
50
51
52
53
54
55
56 public static int length(final Cell cell) {
57 return length(cell.getRowLength(), cell.getFamilyLength(), cell.getQualifierLength(),
58 cell.getValueLength(), cell.getTagsLength(), true);
59 }
60
61 private static int length(short rlen, byte flen, int qlen, int vlen, int tlen, boolean withTags) {
62 if (withTags) {
63 return (int) (KeyValue.getKeyValueDataStructureSize(rlen, flen, qlen, vlen, tlen));
64 }
65 return (int) (KeyValue.getKeyValueDataStructureSize(rlen, flen, qlen, vlen));
66 }
67
68
69
70
71
72
73
74 public static int keyLength(final Cell cell) {
75 return keyLength(cell.getRowLength(), cell.getFamilyLength(), cell.getQualifierLength());
76 }
77
78 private static int keyLength(short rlen, byte flen, int qlen) {
79 return (int) KeyValue.getKeyDataStructureSize(rlen, flen, qlen);
80 }
81
82 public static int lengthWithMvccVersion(final KeyValue kv, final boolean includeMvccVersion) {
83 int length = kv.getLength();
84 if (includeMvccVersion) {
85 length += WritableUtils.getVIntSize(kv.getMvccVersion());
86 }
87 return length;
88 }
89
90 public static int totalLengthWithMvccVersion(final Iterable<? extends KeyValue> kvs,
91 final boolean includeMvccVersion) {
92 int length = 0;
93 for (KeyValue kv : IterableUtils.nullSafe(kvs)) {
94 length += lengthWithMvccVersion(kv, includeMvccVersion);
95 }
96 return length;
97 }
98
99
100
101
102 public static KeyValue copyToNewKeyValue(final Cell cell) {
103 byte[] bytes = copyToNewByteArray(cell);
104 KeyValue kvCell = new KeyValue(bytes, 0, bytes.length);
105 kvCell.setSequenceId(cell.getMvccVersion());
106 return kvCell;
107 }
108
109
110
111
112
113
114 public static ByteBuffer copyKeyToNewByteBuffer(final Cell cell) {
115 byte[] bytes = new byte[keyLength(cell)];
116 appendKeyTo(cell, bytes, 0);
117 ByteBuffer buffer = ByteBuffer.wrap(bytes);
118 return buffer;
119 }
120
121 public static byte[] copyToNewByteArray(final Cell cell) {
122 int v1Length = length(cell);
123 byte[] backingBytes = new byte[v1Length];
124 appendToByteArray(cell, backingBytes, 0);
125 return backingBytes;
126 }
127
128 public static int appendKeyTo(final Cell cell, final byte[] output,
129 final int offset) {
130 int nextOffset = offset;
131 nextOffset = Bytes.putShort(output, nextOffset, cell.getRowLength());
132 nextOffset = CellUtil.copyRowTo(cell, output, nextOffset);
133 nextOffset = Bytes.putByte(output, nextOffset, cell.getFamilyLength());
134 nextOffset = CellUtil.copyFamilyTo(cell, output, nextOffset);
135 nextOffset = CellUtil.copyQualifierTo(cell, output, nextOffset);
136 nextOffset = Bytes.putLong(output, nextOffset, cell.getTimestamp());
137 nextOffset = Bytes.putByte(output, nextOffset, cell.getTypeByte());
138 return nextOffset;
139 }
140
141
142
143
144 public static int appendToByteArray(final Cell cell, final byte[] output, final int offset) {
145
146
147 int pos = offset;
148 pos = Bytes.putInt(output, pos, keyLength(cell));
149 pos = Bytes.putInt(output, pos, cell.getValueLength());
150 pos = appendKeyTo(cell, output, pos);
151 pos = CellUtil.copyValueTo(cell, output, pos);
152 if ((cell.getTagsLength() > 0)) {
153 pos = Bytes.putAsShort(output, pos, cell.getTagsLength());
154 pos = CellUtil.copyTagTo(cell, output, pos);
155 }
156 return pos;
157 }
158
159
160
161
162
163
164 public static ByteBuffer copyToNewByteBuffer(final Cell cell) {
165 byte[] bytes = new byte[length(cell)];
166 appendToByteArray(cell, bytes, 0);
167 ByteBuffer buffer = ByteBuffer.wrap(bytes);
168 return buffer;
169 }
170
171 public static void appendToByteBuffer(final ByteBuffer bb, final KeyValue kv,
172 final boolean includeMvccVersion) {
173
174 bb.limit(bb.position() + kv.getLength());
175 bb.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
176 if (includeMvccVersion) {
177 int numMvccVersionBytes = WritableUtils.getVIntSize(kv.getMvccVersion());
178 ByteBufferUtils.extendLimit(bb, numMvccVersionBytes);
179 ByteBufferUtils.writeVLong(bb, kv.getMvccVersion());
180 }
181 }
182
183
184
185
186
187
188
189
190
191
192
193 public static KeyValue nextShallowCopy(final ByteBuffer bb, final boolean includesMvccVersion,
194 boolean includesTags) {
195 if (bb.isDirect()) {
196 throw new IllegalArgumentException("only supports heap buffers");
197 }
198 if (bb.remaining() < 1) {
199 return null;
200 }
201 KeyValue keyValue = null;
202 int underlyingArrayOffset = bb.arrayOffset() + bb.position();
203 int keyLength = bb.getInt();
204 int valueLength = bb.getInt();
205 ByteBufferUtils.skip(bb, keyLength + valueLength);
206 int tagsLength = 0;
207 if (includesTags) {
208
209 tagsLength = ((bb.get() & 0xff) << 8) ^ (bb.get() & 0xff);
210 ByteBufferUtils.skip(bb, tagsLength);
211 }
212 int kvLength = (int) KeyValue.getKeyValueDataStructureSize(keyLength, valueLength, tagsLength);
213 keyValue = new KeyValue(bb.array(), underlyingArrayOffset, kvLength);
214 if (includesMvccVersion) {
215 long mvccVersion = ByteBufferUtils.readVLong(bb);
216 keyValue.setSequenceId(mvccVersion);
217 }
218 return keyValue;
219 }
220
221
222
223
224
225
226
227 public static KeyValue createFirstKeyInNextRow(final Cell in){
228 byte[] nextRow = new byte[in.getRowLength() + 1];
229 System.arraycopy(in.getRowArray(), in.getRowOffset(), nextRow, 0, in.getRowLength());
230 nextRow[nextRow.length - 1] = 0;
231 return createFirstOnRow(nextRow);
232 }
233
234
235
236
237 public static KeyValue createFirstKeyInIncrementedRow(final Cell in){
238 byte[] thisRow = new SimpleMutableByteRange(in.getRowArray(), in.getRowOffset(),
239 in.getRowLength()).deepCopyToNewArray();
240 byte[] nextRow = Bytes.unsignedCopyAndIncrement(thisRow);
241 return createFirstOnRow(nextRow);
242 }
243
244
245
246
247
248
249
250
251 public static KeyValue previousKey(final KeyValue in) {
252 return createFirstOnRow(CellUtil.cloneRow(in), CellUtil.cloneFamily(in),
253 CellUtil.cloneQualifier(in), in.getTimestamp() - 1);
254 }
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282 public static KeyValue createLastOnRow(final byte[] row, final int roffset, final int rlength,
283 final byte[] family, final int foffset, final int flength, final byte[] qualifier,
284 final int qoffset, final int qlength) {
285 return new KeyValue(row, roffset, rlength, family, foffset, flength, qualifier, qoffset,
286 qlength, HConstants.OLDEST_TIMESTAMP, Type.Minimum, null, 0, 0);
287 }
288
289
290
291
292
293
294
295 public static KeyValue createLastOnRow(Cell kv) {
296 return createLastOnRow(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), null, 0, 0,
297 null, 0, 0);
298 }
299
300
301
302
303
304
305
306
307
308
309
310
311 public static KeyValue createLastOnRowCol(Cell kv) {
312 return new KeyValue(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
313 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), kv.getQualifierArray(),
314 kv.getQualifierOffset(), kv.getQualifierLength(), HConstants.OLDEST_TIMESTAMP,
315 Type.Minimum, null, 0, 0);
316 }
317
318
319
320
321
322
323
324
325
326
327
328 public static KeyValue createFirstOnRowColTS(Cell kv, long ts) {
329 return new KeyValue(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
330 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), kv.getQualifierArray(),
331 kv.getQualifierOffset(), kv.getQualifierLength(), ts, Type.Maximum, kv.getValueArray(),
332 kv.getValueOffset(), kv.getValueLength());
333 }
334
335
336
337
338
339
340
341
342
343 public static KeyValue createFirstOnRow(final byte [] row, int roffset, short rlength) {
344 return new KeyValue(row, roffset, rlength,
345 null, 0, 0, null, 0, 0, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
346 }
347
348
349
350
351
352
353
354
355
356 public static KeyValue createLastOnRow(final byte[] row) {
357 return new KeyValue(row, null, null, HConstants.LATEST_TIMESTAMP, Type.Minimum);
358 }
359
360
361
362
363
364
365
366
367
368 public static KeyValue createFirstOnRow(final byte [] row) {
369 return createFirstOnRow(row, HConstants.LATEST_TIMESTAMP);
370 }
371
372
373
374
375
376
377
378
379 public static KeyValue createFirstOnRow(final byte [] row,
380 final long ts) {
381 return new KeyValue(row, null, null, ts, Type.Maximum);
382 }
383
384
385
386
387
388
389
390
391
392
393 public static KeyValue createFirstOnRow(final byte [] row, final byte [] family,
394 final byte [] qualifier) {
395 return new KeyValue(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum);
396 }
397
398
399
400
401
402
403
404
405
406
407 public static KeyValue createFirstDeleteFamilyOnRow(final byte [] row,
408 final byte [] family) {
409 return new KeyValue(row, family, null, HConstants.LATEST_TIMESTAMP,
410 Type.DeleteFamily);
411 }
412
413
414
415
416
417
418
419
420 public static KeyValue createFirstOnRow(final byte [] row, final byte [] f,
421 final byte [] q, final long ts) {
422 return new KeyValue(row, f, q, ts, Type.Maximum);
423 }
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441 public static KeyValue createFirstOnRow(final byte [] row,
442 final int roffset, final int rlength, final byte [] family,
443 final int foffset, final int flength, final byte [] qualifier,
444 final int qoffset, final int qlength) {
445 return new KeyValue(row, roffset, rlength, family,
446 foffset, flength, qualifier, qoffset, qlength,
447 HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
448 }
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466 public static KeyValue createFirstOnRow(byte [] buffer, final byte [] row,
467 final byte [] family, final byte [] qualifier)
468 throws IllegalArgumentException {
469 return createFirstOnRow(buffer, 0, row, 0, row.length,
470 family, 0, family.length,
471 qualifier, 0, qualifier.length);
472 }
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497 public static KeyValue createFirstOnRow(byte[] buffer, final int boffset, final byte[] row,
498 final int roffset, final int rlength, final byte[] family, final int foffset,
499 final int flength, final byte[] qualifier, final int qoffset, final int qlength)
500 throws IllegalArgumentException {
501
502 long lLength = KeyValue.getKeyValueDataStructureSize(rlength, flength, qlength, 0);
503
504 if (lLength > Integer.MAX_VALUE) {
505 throw new IllegalArgumentException("KeyValue length " + lLength + " > " + Integer.MAX_VALUE);
506 }
507 int iLength = (int) lLength;
508 if (buffer.length - boffset < iLength) {
509 throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < "
510 + iLength);
511 }
512
513 int len = KeyValue.writeByteArray(buffer, boffset, row, roffset, rlength, family, foffset,
514 flength, qualifier, qoffset, qlength, HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum,
515 null, 0, 0, null);
516 return new KeyValue(buffer, boffset, len);
517 }
518
519
520
521
522
523
524
525
526 public static KeyValue createFirstOnRowColTS(KeyValue kv, long ts) {
527 return new KeyValue(
528 kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
529 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(),
530 kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength(),
531 ts, Type.Maximum, kv.getValueArray(), kv.getValueOffset(), kv.getValueLength());
532 }
533
534
535
536
537
538
539
540
541 @Deprecated
542 public static KeyValue ensureKeyValue(final Cell cell) {
543 if (cell == null) return null;
544 return cell instanceof KeyValue? (KeyValue)cell: copyToNewKeyValue(cell);
545 }
546
547
548
549
550
551
552
553
554
555
556
557 @Deprecated
558 public static KeyValue ensureKeyValueTypeForMR(final Cell cell) {
559 if (cell == null) return null;
560 if (cell instanceof KeyValue) {
561 if (cell.getClass().getName().equals(KeyValue.class.getName())) {
562 return (KeyValue) cell;
563 }
564
565
566 KeyValue kv = (KeyValue) cell;
567 KeyValue newKv = new KeyValue(kv.bytes, kv.offset, kv.length);
568 newKv.setSequenceId(kv.getSequenceId());
569 return newKv;
570 }
571 return copyToNewKeyValue(cell);
572 }
573
574 @Deprecated
575 public static List<KeyValue> ensureKeyValues(List<Cell> cells) {
576 List<KeyValue> lazyList = Lists.transform(cells, new Function<Cell, KeyValue>() {
577 @Override
578 public KeyValue apply(Cell arg0) {
579 return KeyValueUtil.ensureKeyValue(arg0);
580 }
581 });
582 return new ArrayList<KeyValue>(lazyList);
583 }
584
585
586
587
588
589
590
591
592
593
594
595
596
597 public static KeyValue iscreate(final InputStream in, boolean withTags) throws IOException {
598 byte[] intBytes = new byte[Bytes.SIZEOF_INT];
599 int bytesRead = 0;
600 while (bytesRead < intBytes.length) {
601 int n = in.read(intBytes, bytesRead, intBytes.length - bytesRead);
602 if (n < 0) {
603 if (bytesRead == 0) {
604 throw new EOFException();
605 }
606 throw new IOException("Failed read of int, read " + bytesRead + " bytes");
607 }
608 bytesRead += n;
609 }
610
611 byte[] bytes = new byte[Bytes.toInt(intBytes)];
612 IOUtils.readFully(in, bytes, 0, bytes.length);
613 if (withTags) {
614 return new KeyValue(bytes, 0, bytes.length);
615 } else {
616 return new NoTagsKeyValue(bytes, 0, bytes.length);
617 }
618 }
619
620 public static void oswrite(final Cell cell, final OutputStream out, final boolean withTags)
621 throws IOException {
622 if (cell instanceof KeyValue) {
623 KeyValue.oswrite((KeyValue) cell, out, withTags);
624 } else {
625 short rlen = cell.getRowLength();
626 byte flen = cell.getFamilyLength();
627 int qlen = cell.getQualifierLength();
628 int vlen = cell.getValueLength();
629 int tlen = cell.getTagsLength();
630
631
632 StreamUtils.writeInt(out, length(rlen, flen, qlen, vlen, tlen, withTags));
633
634 StreamUtils.writeInt(out, keyLength(rlen, flen, qlen));
635
636 StreamUtils.writeInt(out, vlen);
637
638 StreamUtils.writeShort(out, rlen);
639 out.write(cell.getRowArray(), cell.getRowOffset(), rlen);
640
641 out.write(flen);
642 out.write(cell.getFamilyArray(), cell.getFamilyOffset(), flen);
643
644 out.write(cell.getQualifierArray(), cell.getQualifierOffset(), qlen);
645
646 StreamUtils.writeLong(out, cell.getTimestamp());
647
648 out.write(cell.getTypeByte());
649
650 out.write(cell.getValueArray(), cell.getValueOffset(), vlen);
651
652 if (withTags && tlen > 0) {
653
654
655
656 out.write((byte) (0xff & (tlen >> 8)));
657 out.write((byte) (0xff & tlen));
658 out.write(cell.getTagsArray(), cell.getTagsOffset(), tlen);
659 }
660 }
661 }
662 }