1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import java.io.IOException;
22 import java.io.OutputStream;
23 import java.nio.ByteBuffer;
24 import java.util.ArrayList;
25 import java.util.List;
26
27 import org.apache.hadoop.hbase.classification.InterfaceAudience;
28 import org.apache.hadoop.hbase.KeyValue.Type;
29 import org.apache.hadoop.hbase.io.util.StreamUtils;
30 import org.apache.hadoop.hbase.util.ByteBufferUtils;
31 import org.apache.hadoop.hbase.util.Bytes;
32 import org.apache.hadoop.hbase.util.IterableUtils;
33 import org.apache.hadoop.hbase.util.SimpleMutableByteRange;
34 import org.apache.hadoop.io.WritableUtils;
35
36 import com.google.common.base.Function;
37 import com.google.common.collect.Lists;
38
39
40
41
42 @InterfaceAudience.Private
43 public class KeyValueUtil {
44
45
46
47
48
49
50
51
52 public static int length(final Cell cell) {
53 return length(cell.getRowLength(), cell.getFamilyLength(), cell.getQualifierLength(),
54 cell.getValueLength(), cell.getTagsLength(), true);
55 }
56
57 private static int length(short rlen, byte flen, int qlen, int vlen, int tlen, boolean withTags) {
58 if (withTags) {
59 return (int) (KeyValue.getKeyValueDataStructureSize(rlen, flen, qlen, vlen, tlen));
60 }
61 return (int) (KeyValue.getKeyValueDataStructureSize(rlen, flen, qlen, vlen));
62 }
63
64
65
66
67
68
69
70 public static int keyLength(final Cell cell) {
71 return keyLength(cell.getRowLength(), cell.getFamilyLength(), cell.getQualifierLength());
72 }
73
74 private static int keyLength(short rlen, byte flen, int qlen) {
75 return (int) KeyValue.getKeyDataStructureSize(rlen, flen, qlen);
76 }
77
78 public static int lengthWithMvccVersion(final KeyValue kv, final boolean includeMvccVersion) {
79 int length = kv.getLength();
80 if (includeMvccVersion) {
81 length += WritableUtils.getVIntSize(kv.getMvccVersion());
82 }
83 return length;
84 }
85
86 public static int totalLengthWithMvccVersion(final Iterable<? extends KeyValue> kvs,
87 final boolean includeMvccVersion) {
88 int length = 0;
89 for (KeyValue kv : IterableUtils.nullSafe(kvs)) {
90 length += lengthWithMvccVersion(kv, includeMvccVersion);
91 }
92 return length;
93 }
94
95
96
97
98 public static KeyValue copyToNewKeyValue(final Cell cell) {
99 byte[] bytes = copyToNewByteArray(cell);
100 KeyValue kvCell = new KeyValue(bytes, 0, bytes.length);
101 kvCell.setSequenceId(cell.getMvccVersion());
102 return kvCell;
103 }
104
105
106
107
108
109
110 public static ByteBuffer copyKeyToNewByteBuffer(final Cell cell) {
111 byte[] bytes = new byte[keyLength(cell)];
112 appendKeyTo(cell, bytes, 0);
113 ByteBuffer buffer = ByteBuffer.wrap(bytes);
114 return buffer;
115 }
116
117 public static byte[] copyToNewByteArray(final Cell cell) {
118 int v1Length = length(cell);
119 byte[] backingBytes = new byte[v1Length];
120 appendToByteArray(cell, backingBytes, 0);
121 return backingBytes;
122 }
123
124 public static int appendKeyTo(final Cell cell, final byte[] output,
125 final int offset) {
126 int nextOffset = offset;
127 nextOffset = Bytes.putShort(output, nextOffset, cell.getRowLength());
128 nextOffset = CellUtil.copyRowTo(cell, output, nextOffset);
129 nextOffset = Bytes.putByte(output, nextOffset, cell.getFamilyLength());
130 nextOffset = CellUtil.copyFamilyTo(cell, output, nextOffset);
131 nextOffset = CellUtil.copyQualifierTo(cell, output, nextOffset);
132 nextOffset = Bytes.putLong(output, nextOffset, cell.getTimestamp());
133 nextOffset = Bytes.putByte(output, nextOffset, cell.getTypeByte());
134 return nextOffset;
135 }
136
137
138
139
140 public static int appendToByteArray(final Cell cell, final byte[] output, final int offset) {
141
142
143 int pos = offset;
144 pos = Bytes.putInt(output, pos, keyLength(cell));
145 pos = Bytes.putInt(output, pos, cell.getValueLength());
146 pos = appendKeyTo(cell, output, pos);
147 pos = CellUtil.copyValueTo(cell, output, pos);
148 if ((cell.getTagsLength() > 0)) {
149 pos = Bytes.putAsShort(output, pos, cell.getTagsLength());
150 pos = CellUtil.copyTagTo(cell, output, pos);
151 }
152 return pos;
153 }
154
155
156
157
158
159
160 public static ByteBuffer copyToNewByteBuffer(final Cell cell) {
161 byte[] bytes = new byte[length(cell)];
162 appendToByteArray(cell, bytes, 0);
163 ByteBuffer buffer = ByteBuffer.wrap(bytes);
164 return buffer;
165 }
166
167 public static void appendToByteBuffer(final ByteBuffer bb, final KeyValue kv,
168 final boolean includeMvccVersion) {
169
170 bb.limit(bb.position() + kv.getLength());
171 bb.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
172 if (includeMvccVersion) {
173 int numMvccVersionBytes = WritableUtils.getVIntSize(kv.getMvccVersion());
174 ByteBufferUtils.extendLimit(bb, numMvccVersionBytes);
175 ByteBufferUtils.writeVLong(bb, kv.getMvccVersion());
176 }
177 }
178
179
180
181
182
183
184
185
186
187
188
189 public static KeyValue nextShallowCopy(final ByteBuffer bb, final boolean includesMvccVersion,
190 boolean includesTags) {
191 if (bb.isDirect()) {
192 throw new IllegalArgumentException("only supports heap buffers");
193 }
194 if (bb.remaining() < 1) {
195 return null;
196 }
197 KeyValue keyValue = null;
198 int underlyingArrayOffset = bb.arrayOffset() + bb.position();
199 int keyLength = bb.getInt();
200 int valueLength = bb.getInt();
201 ByteBufferUtils.skip(bb, keyLength + valueLength);
202 int tagsLength = 0;
203 if (includesTags) {
204
205 tagsLength = ((bb.get() & 0xff) << 8) ^ (bb.get() & 0xff);
206 ByteBufferUtils.skip(bb, tagsLength);
207 }
208 int kvLength = (int) KeyValue.getKeyValueDataStructureSize(keyLength, valueLength, tagsLength);
209 keyValue = new KeyValue(bb.array(), underlyingArrayOffset, kvLength);
210 if (includesMvccVersion) {
211 long mvccVersion = ByteBufferUtils.readVLong(bb);
212 keyValue.setSequenceId(mvccVersion);
213 }
214 return keyValue;
215 }
216
217
218
219
220
221
222
223 public static KeyValue createFirstKeyInNextRow(final Cell in){
224 byte[] nextRow = new byte[in.getRowLength() + 1];
225 System.arraycopy(in.getRowArray(), in.getRowOffset(), nextRow, 0, in.getRowLength());
226 nextRow[nextRow.length - 1] = 0;
227 return createFirstOnRow(nextRow);
228 }
229
230
231
232
233 public static KeyValue createFirstKeyInIncrementedRow(final Cell in){
234 byte[] thisRow = new SimpleMutableByteRange(in.getRowArray(), in.getRowOffset(),
235 in.getRowLength()).deepCopyToNewArray();
236 byte[] nextRow = Bytes.unsignedCopyAndIncrement(thisRow);
237 return createFirstOnRow(nextRow);
238 }
239
240
241
242
243
244
245
246
247 public static KeyValue previousKey(final KeyValue in) {
248 return createFirstOnRow(CellUtil.cloneRow(in), CellUtil.cloneFamily(in),
249 CellUtil.cloneQualifier(in), in.getTimestamp() - 1);
250 }
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278 public static KeyValue createLastOnRow(final byte[] row, final int roffset, final int rlength,
279 final byte[] family, final int foffset, final int flength, final byte[] qualifier,
280 final int qoffset, final int qlength) {
281 return new KeyValue(row, roffset, rlength, family, foffset, flength, qualifier, qoffset,
282 qlength, HConstants.OLDEST_TIMESTAMP, Type.Minimum, null, 0, 0);
283 }
284
285
286
287
288
289
290
291 public static KeyValue createLastOnRow(Cell kv) {
292 return createLastOnRow(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), null, 0, 0,
293 null, 0, 0);
294 }
295
296
297
298
299
300
301
302
303
304
305
306
307 public static KeyValue createLastOnRowCol(Cell kv) {
308 return new KeyValue(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
309 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), kv.getQualifierArray(),
310 kv.getQualifierOffset(), kv.getQualifierLength(), HConstants.OLDEST_TIMESTAMP,
311 Type.Minimum, null, 0, 0);
312 }
313
314
315
316
317
318
319
320
321
322
323
324 public static KeyValue createFirstOnRowColTS(Cell kv, long ts) {
325 return new KeyValue(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
326 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), kv.getQualifierArray(),
327 kv.getQualifierOffset(), kv.getQualifierLength(), ts, Type.Maximum, kv.getValueArray(),
328 kv.getValueOffset(), kv.getValueLength());
329 }
330
331
332
333
334
335
336
337
338
339 public static KeyValue createFirstOnRow(final byte [] row, int roffset, short rlength) {
340 return new KeyValue(row, roffset, rlength,
341 null, 0, 0, null, 0, 0, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
342 }
343
344
345
346
347
348
349
350
351
352 public static KeyValue createLastOnRow(final byte[] row) {
353 return new KeyValue(row, null, null, HConstants.LATEST_TIMESTAMP, Type.Minimum);
354 }
355
356
357
358
359
360
361
362
363
364 public static KeyValue createFirstOnRow(final byte [] row) {
365 return createFirstOnRow(row, HConstants.LATEST_TIMESTAMP);
366 }
367
368
369
370
371
372
373
374
375 public static KeyValue createFirstOnRow(final byte [] row,
376 final long ts) {
377 return new KeyValue(row, null, null, ts, Type.Maximum);
378 }
379
380
381
382
383
384
385
386
387
388
389 public static KeyValue createFirstOnRow(final byte [] row, final byte [] family,
390 final byte [] qualifier) {
391 return new KeyValue(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum);
392 }
393
394
395
396
397
398
399
400
401
402
403 public static KeyValue createFirstDeleteFamilyOnRow(final byte [] row,
404 final byte [] family) {
405 return new KeyValue(row, family, null, HConstants.LATEST_TIMESTAMP,
406 Type.DeleteFamily);
407 }
408
409
410
411
412
413
414
415
416 public static KeyValue createFirstOnRow(final byte [] row, final byte [] f,
417 final byte [] q, final long ts) {
418 return new KeyValue(row, f, q, ts, Type.Maximum);
419 }
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437 public static KeyValue createFirstOnRow(final byte [] row,
438 final int roffset, final int rlength, final byte [] family,
439 final int foffset, final int flength, final byte [] qualifier,
440 final int qoffset, final int qlength) {
441 return new KeyValue(row, roffset, rlength, family,
442 foffset, flength, qualifier, qoffset, qlength,
443 HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
444 }
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462 public static KeyValue createFirstOnRow(byte [] buffer, final byte [] row,
463 final byte [] family, final byte [] qualifier)
464 throws IllegalArgumentException {
465 return createFirstOnRow(buffer, 0, row, 0, row.length,
466 family, 0, family.length,
467 qualifier, 0, qualifier.length);
468 }
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493 public static KeyValue createFirstOnRow(byte[] buffer, final int boffset, final byte[] row,
494 final int roffset, final int rlength, final byte[] family, final int foffset,
495 final int flength, final byte[] qualifier, final int qoffset, final int qlength)
496 throws IllegalArgumentException {
497
498 long lLength = KeyValue.getKeyValueDataStructureSize(rlength, flength, qlength, 0);
499
500 if (lLength > Integer.MAX_VALUE) {
501 throw new IllegalArgumentException("KeyValue length " + lLength + " > " + Integer.MAX_VALUE);
502 }
503 int iLength = (int) lLength;
504 if (buffer.length - boffset < iLength) {
505 throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < "
506 + iLength);
507 }
508
509 int len = KeyValue.writeByteArray(buffer, boffset, row, roffset, rlength, family, foffset,
510 flength, qualifier, qoffset, qlength, HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum,
511 null, 0, 0, null);
512 return new KeyValue(buffer, boffset, len);
513 }
514
515
516
517
518
519
520
521
522 public static KeyValue createFirstOnRowColTS(KeyValue kv, long ts) {
523 return new KeyValue(
524 kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
525 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(),
526 kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength(),
527 ts, Type.Maximum, kv.getValueArray(), kv.getValueOffset(), kv.getValueLength());
528 }
529
530
531
532
533
534
535
536
537 @Deprecated
538 public static KeyValue ensureKeyValue(final Cell cell) {
539 if (cell == null) return null;
540 return cell instanceof KeyValue? (KeyValue)cell: copyToNewKeyValue(cell);
541 }
542
543 @Deprecated
544 public static List<KeyValue> ensureKeyValues(List<Cell> cells) {
545 List<KeyValue> lazyList = Lists.transform(cells, new Function<Cell, KeyValue>() {
546 @Override
547 public KeyValue apply(Cell arg0) {
548 return KeyValueUtil.ensureKeyValue(arg0);
549 }
550 });
551 return new ArrayList<KeyValue>(lazyList);
552 }
553
554 public static void oswrite(final Cell cell, final OutputStream out, final boolean withTags)
555 throws IOException {
556 if (cell instanceof KeyValue) {
557 KeyValue.oswrite((KeyValue) cell, out, withTags);
558 } else {
559 short rlen = cell.getRowLength();
560 byte flen = cell.getFamilyLength();
561 int qlen = cell.getQualifierLength();
562 int vlen = cell.getValueLength();
563 int tlen = cell.getTagsLength();
564
565
566 StreamUtils.writeInt(out, length(rlen, flen, qlen, vlen, tlen, withTags));
567
568 StreamUtils.writeInt(out, keyLength(rlen, flen, qlen));
569
570 StreamUtils.writeInt(out, vlen);
571
572 StreamUtils.writeShort(out, rlen);
573 out.write(cell.getRowArray(), cell.getRowOffset(), rlen);
574
575 out.write(flen);
576 out.write(cell.getFamilyArray(), cell.getFamilyOffset(), flen);
577
578 out.write(cell.getQualifierArray(), cell.getQualifierOffset(), qlen);
579
580 StreamUtils.writeLong(out, cell.getTimestamp());
581
582 out.write(cell.getTypeByte());
583
584 out.write(cell.getValueArray(), cell.getValueOffset(), vlen);
585
586 if (withTags && tlen > 0) {
587
588
589
590 out.write((byte) (0xff & (tlen >> 8)));
591 out.write((byte) (0xff & tlen));
592 out.write(cell.getTagsArray(), cell.getTagsOffset(), tlen);
593 }
594 }
595 }
596 }