View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.codec.prefixtree;
20  
21  import java.nio.ByteBuffer;
22  
23  import org.apache.hadoop.hbase.ByteBufferedCell;
24  import org.apache.hadoop.hbase.Cell;
25  import org.apache.hadoop.hbase.CellComparator;
26  import org.apache.hadoop.hbase.CellUtil;
27  import org.apache.hadoop.hbase.KeyValue.Type;
28  import org.apache.hadoop.hbase.SettableSequenceId;
29  import org.apache.hadoop.hbase.classification.InterfaceAudience;
30  import org.apache.hadoop.hbase.codec.prefixtree.decode.DecoderFactory;
31  import org.apache.hadoop.hbase.codec.prefixtree.decode.PrefixTreeArraySearcher;
32  import org.apache.hadoop.hbase.codec.prefixtree.scanner.CellScannerPosition;
33  import org.apache.hadoop.hbase.io.HeapSize;
34  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder.EncodedSeeker;
35  import org.apache.hadoop.hbase.nio.ByteBuff;
36  import org.apache.hadoop.hbase.util.ByteBufferUtils;
37  import org.apache.hadoop.hbase.util.Bytes;
38  import org.apache.hadoop.hbase.util.ClassSize;
39  
40  /**
41   * These methods have the same definition as any implementation of the EncodedSeeker.
42   *
43   * In the future, the EncodedSeeker could be modified to work with the Cell interface directly.  It
44   * currently returns a new KeyValue object each time getKeyValue is called.  This is not horrible,
45   * but in order to create a new KeyValue object, we must first allocate a new byte[] and copy in
46   * the data from the PrefixTreeCell.  It is somewhat heavyweight right now.
47   */
48  @InterfaceAudience.Private
49  public class PrefixTreeSeeker implements EncodedSeeker {
50  
51    protected ByteBuffer block;
52    protected boolean includeMvccVersion;
53    protected PrefixTreeArraySearcher ptSearcher;
54  
55    public PrefixTreeSeeker(boolean includeMvccVersion) {
56      this.includeMvccVersion = includeMvccVersion;
57    }
58  
59    @Override
60    public void setCurrentBuffer(ByteBuff fullBlockBuffer) {
61      ptSearcher = DecoderFactory.checkOut(fullBlockBuffer, includeMvccVersion);
62      rewind();
63    }
64  
65    /**
66     * <p>
67     * Currently unused.
68     * </p>
69     * TODO performance leak. should reuse the searchers. hbase does not currently have a hook where
70     * this can be called
71     */
72    public void releaseCurrentSearcher(){
73      DecoderFactory.checkIn(ptSearcher);
74    }
75  
76  
77    @Override
78    public Cell getKey() {
79      return ptSearcher.current();
80    }
81  
82  
83    @Override
84    public ByteBuffer getValueShallowCopy() {
85      return CellUtil.getValueBufferShallowCopy(ptSearcher.current());
86    }
87  
88    /**
89     * currently must do deep copy into new array
90     */
91    @Override
92    public Cell getCell() {
93      // The PrefixTreecell is of type BytebufferedCell and the value part of the cell
94      // determines whether we are offheap cell or onheap cell.  All other parts of the cell-
95      // row, fam and col are all represented as onheap byte[]
96      ByteBufferedCell cell = (ByteBufferedCell)ptSearcher.current();
97      if (cell == null) {
98        return null;
99      }
100     // Use the ByteBuffered cell to see if the Cell is onheap or offheap
101     if (cell.getValueByteBuffer().hasArray()) {
102       return new OnheapPrefixTreeCell(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
103           cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
104           cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
105           cell.getValueArray(), cell.getValueOffset(), cell.getValueLength(), cell.getTagsArray(),
106           cell.getTagsOffset(), cell.getTagsLength(), cell.getTimestamp(), cell.getTypeByte(),
107           cell.getSequenceId());
108     } else {
109       return new OffheapPrefixTreeCell(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
110           cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
111           cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
112           cell.getValueByteBuffer(), cell.getValuePosition(), cell.getValueLength(),
113           cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength(), cell.getTimestamp(),
114           cell.getTypeByte(), cell.getSequenceId());
115     }
116   }
117 
118   /**
119    * <p>
120    * Currently unused.
121    * </p><p>
122    * A nice, lightweight reference, though the underlying cell is transient. This method may return
123    * the same reference to the backing PrefixTreeCell repeatedly, while other implementations may
124    * return a different reference for each Cell.
125    * </p>
126    * The goal will be to transition the upper layers of HBase, like Filters and KeyValueHeap, to
127    * use this method instead of the getKeyValue() methods above.
128    */
129   public Cell get() {
130     return ptSearcher.current();
131   }
132 
133   @Override
134   public void rewind() {
135     ptSearcher.positionAtFirstCell();
136   }
137 
138   @Override
139   public boolean next() {
140     return ptSearcher.advance();
141   }
142 
143   public boolean advance() {
144     return ptSearcher.advance();
145   }
146 
147 
148   private static final boolean USE_POSITION_BEFORE = false;
149 
150   /*
151    * Support both of these options since the underlying PrefixTree supports
152    * both. Possibly expand the EncodedSeeker to utilize them both.
153    */
154 
155   protected int seekToOrBeforeUsingPositionAtOrBefore(Cell kv, boolean seekBefore) {
156     // this does a deep copy of the key byte[] because the CellSearcher
157     // interface wants a Cell
158     CellScannerPosition position = ptSearcher.seekForwardToOrBefore(kv);
159 
160     if (CellScannerPosition.AT == position) {
161       if (seekBefore) {
162         ptSearcher.previous();
163         return 1;
164       }
165       return 0;
166     }
167 
168     return 1;
169   }
170 
171   protected int seekToOrBeforeUsingPositionAtOrAfter(Cell kv, boolean seekBefore) {
172     // should probably switch this to use the seekForwardToOrBefore method
173     CellScannerPosition position = ptSearcher.seekForwardToOrAfter(kv);
174 
175     if (CellScannerPosition.AT == position) {
176       if (seekBefore) {
177         ptSearcher.previous();
178         return 1;
179       }
180       return 0;
181 
182     }
183 
184     if (CellScannerPosition.AFTER == position) {
185       if (!ptSearcher.isBeforeFirst()) {
186         ptSearcher.previous();
187       }
188       return 1;
189     }
190 
191     if (position == CellScannerPosition.AFTER_LAST) {
192       if (seekBefore) {
193         ptSearcher.previous();
194       }
195       return 1;
196     }
197 
198     throw new RuntimeException("unexpected CellScannerPosition:" + position);
199   }
200 
201   @Override
202   public int seekToKeyInBlock(Cell key, boolean forceBeforeOnExactMatch) {
203     if (USE_POSITION_BEFORE) {
204       return seekToOrBeforeUsingPositionAtOrBefore(key, forceBeforeOnExactMatch);
205     } else {
206       return seekToOrBeforeUsingPositionAtOrAfter(key, forceBeforeOnExactMatch);
207     }
208   }
209 
210   @Override
211   public int compareKey(CellComparator comparator, Cell key) {
212     return comparator.compare(key,
213         ptSearcher.current());
214   }
215 
216   /**
217    * Cloned version of the PrefixTreeCell where except the value part, the rest
218    * of the key part is deep copied
219    *
220    */
221   private static class OnheapPrefixTreeCell implements Cell, SettableSequenceId, HeapSize {
222     private static final long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT
223         + (5 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_LONG) + (4 * Bytes.SIZEOF_INT)
224         + (Bytes.SIZEOF_SHORT) + (2 * Bytes.SIZEOF_BYTE) + (5 * ClassSize.ARRAY));
225     private byte[] row;
226     private short rowLength;
227     private byte[] fam;
228     private byte famLength;
229     private byte[] qual;
230     private int qualLength;
231     private byte[] val;
232     private int valOffset;
233     private int valLength;
234     private byte[] tag;
235     private int tagsLength;
236     private long ts;
237     private long seqId;
238     private byte type;
239 
240     public OnheapPrefixTreeCell(byte[] row, int rowOffset, short rowLength, byte[] fam,
241         int famOffset, byte famLength, byte[] qual, int qualOffset, int qualLength, byte[] val,
242         int valOffset, int valLength, byte[] tag, int tagOffset, int tagLength, long ts, byte type,
243         long seqId) {
244       this.row = new byte[rowLength];
245       System.arraycopy(row, rowOffset, this.row, 0, rowLength);
246       this.rowLength = rowLength;
247       this.fam = new byte[famLength];
248       System.arraycopy(fam, famOffset, this.fam, 0, famLength);
249       this.famLength = famLength;
250       this.qual = new byte[qualLength];
251       System.arraycopy(qual, qualOffset, this.qual, 0, qualLength);
252       this.qualLength = qualLength;
253       this.tag = new byte[tagLength];
254       System.arraycopy(tag, tagOffset, this.tag, 0, tagLength);
255       this.tagsLength = tagLength;
256       this.val = val;
257       this.valLength = valLength;
258       this.valOffset = valOffset;
259       this.ts = ts;
260       this.seqId = seqId;
261       this.type = type;
262     }
263 
264     @Override
265     public void setSequenceId(long seqId) {
266       this.seqId = seqId;
267     }
268 
269     @Override
270     public byte[] getRowArray() {
271       return this.row;
272     }
273 
274     @Override
275     public int getRowOffset() {
276       return 0;
277     }
278 
279     @Override
280     public short getRowLength() {
281       return this.rowLength;
282     }
283 
284     @Override
285     public byte[] getFamilyArray() {
286       return this.fam;
287     }
288 
289     @Override
290     public int getFamilyOffset() {
291       return 0;
292     }
293 
294     @Override
295     public byte getFamilyLength() {
296       return this.famLength;
297     }
298 
299     @Override
300     public byte[] getQualifierArray() {
301       return this.qual;
302     }
303 
304     @Override
305     public int getQualifierOffset() {
306       return 0;
307     }
308 
309     @Override
310     public int getQualifierLength() {
311       return this.qualLength;
312     }
313 
314     @Override
315     public long getTimestamp() {
316       return ts;
317     }
318 
319     @Override
320     public byte getTypeByte() {
321       return type;
322     }
323 
324     @Override
325     public long getSequenceId() {
326       return seqId;
327     }
328 
329     @Override
330     public byte[] getValueArray() {
331       return val;
332     }
333 
334     @Override
335     public int getValueOffset() {
336       return this.valOffset;
337     }
338 
339     @Override
340     public int getValueLength() {
341       return this.valLength;
342     }
343 
344     @Override
345     public byte[] getTagsArray() {
346       return this.tag;
347     }
348 
349     @Override
350     public int getTagsOffset() {
351       return 0;
352     }
353 
354     @Override
355     public int getTagsLength() {
356       return this.tagsLength;
357     }
358 
359     @Override
360     public String toString() {
361       String row = Bytes.toStringBinary(getRowArray(), getRowOffset(), getRowLength());
362       String family = Bytes.toStringBinary(getFamilyArray(), getFamilyOffset(), getFamilyLength());
363       String qualifier = Bytes.toStringBinary(getQualifierArray(), getQualifierOffset(),
364           getQualifierLength());
365       String timestamp = String.valueOf((getTimestamp()));
366       return row + "/" + family + (family != null && family.length() > 0 ? ":" : "") + qualifier
367           + "/" + timestamp + "/" + Type.codeToType(type);
368     }
369 
370     @Override
371     public long heapSize() {
372       return FIXED_OVERHEAD + rowLength + famLength + qualLength + valLength + tagsLength;
373     }
374   }
375 
376   private static class OffheapPrefixTreeCell extends ByteBufferedCell implements Cell,
377       SettableSequenceId, HeapSize {
378     private static final long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT
379         + (5 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_LONG) + (4 * Bytes.SIZEOF_INT)
380         + (Bytes.SIZEOF_SHORT) + (2 * Bytes.SIZEOF_BYTE) + (5 * ClassSize.BYTE_BUFFER));
381     private ByteBuffer rowBuff;
382     private short rowLength;
383     private ByteBuffer famBuff;
384     private byte famLength;
385     private ByteBuffer qualBuff;
386     private int qualLength;
387     private ByteBuffer val;
388     private int valOffset;
389     private int valLength;
390     private ByteBuffer tagBuff;
391     private int tagsLength;
392     private long ts;
393     private long seqId;
394     private byte type;
395     public OffheapPrefixTreeCell(byte[] row, int rowOffset, short rowLength, byte[] fam,
396         int famOffset, byte famLength, byte[] qual, int qualOffset, int qualLength, ByteBuffer val,
397         int valOffset, int valLength, byte[] tag, int tagOffset, int tagLength, long ts, byte type,
398         long seqId) {
399       byte[] tmpRow = new byte[rowLength];
400       System.arraycopy(row, rowOffset, tmpRow, 0, rowLength);
401       this.rowBuff = ByteBuffer.wrap(tmpRow);
402       this.rowLength = rowLength;
403       byte[] tmpFam = new byte[famLength];
404       System.arraycopy(fam, famOffset, tmpFam, 0, famLength);
405       this.famBuff = ByteBuffer.wrap(tmpFam);
406       this.famLength = famLength;
407       byte[] tmpQual = new byte[qualLength];
408       System.arraycopy(qual, qualOffset, tmpQual, 0, qualLength);
409       this.qualBuff = ByteBuffer.wrap(tmpQual);
410       this.qualLength = qualLength;
411       byte[] tmpTag = new byte[tagLength];
412       System.arraycopy(tag, tagOffset, tmpTag, 0, tagLength);
413       this.tagBuff = ByteBuffer.wrap(tmpTag);
414       this.tagsLength = tagLength;
415       this.val = val;
416       this.valLength = valLength;
417       this.valOffset = valOffset;
418       this.ts = ts;
419       this.seqId = seqId;
420       this.type = type;
421     }
422     
423     @Override
424     public void setSequenceId(long seqId) {
425       this.seqId = seqId;
426     }
427 
428     @Override
429     public byte[] getRowArray() {
430       return this.rowBuff.array();
431     }
432 
433     @Override
434     public int getRowOffset() {
435       return getRowPosition();
436     }
437 
438     @Override
439     public short getRowLength() {
440       return this.rowLength;
441     }
442 
443     @Override
444     public byte[] getFamilyArray() {
445       return this.famBuff.array();
446     }
447 
448     @Override
449     public int getFamilyOffset() {
450       return getFamilyPosition();
451     }
452 
453     @Override
454     public byte getFamilyLength() {
455       return this.famLength;
456     }
457 
458     @Override
459     public byte[] getQualifierArray() {
460       return this.qualBuff.array();
461     }
462 
463     @Override
464     public int getQualifierOffset() {
465       return getQualifierPosition();
466     }
467 
468     @Override
469     public int getQualifierLength() {
470       return this.qualLength;
471     }
472 
473     @Override
474     public long getTimestamp() {
475       return ts;
476     }
477 
478     @Override
479     public byte getTypeByte() {
480       return type;
481     }
482 
483     @Override
484     public long getSequenceId() {
485       return seqId;
486     }
487 
488     @Override
489     public byte[] getValueArray() {
490       byte[] tmpVal = new byte[valLength];
491       ByteBufferUtils.copyFromBufferToArray(tmpVal, val, valOffset, 0, valLength);
492       return tmpVal;
493     }
494 
495     @Override
496     public int getValueOffset() {
497       return 0;
498     }
499 
500     @Override
501     public int getValueLength() {
502       return this.valLength;
503     }
504 
505     @Override
506     public byte[] getTagsArray() {
507       return this.tagBuff.array();
508     }
509 
510     @Override
511     public int getTagsOffset() {
512       return getTagsPosition();
513     }
514 
515     @Override
516     public int getTagsLength() {
517       return this.tagsLength;
518     }
519     
520     @Override
521     public ByteBuffer getRowByteBuffer() {
522       return this.rowBuff;
523     }
524     
525     @Override
526     public int getRowPosition() {
527       return 0;
528     }
529     
530     @Override
531     public ByteBuffer getFamilyByteBuffer() {
532       return this.famBuff;
533     }
534     
535     @Override
536     public int getFamilyPosition() {
537       return 0;
538     }
539     
540     @Override
541     public ByteBuffer getQualifierByteBuffer() {
542       return this.qualBuff;
543     }
544 
545     @Override
546     public int getQualifierPosition() {
547       return 0;
548     }
549 
550     @Override
551     public ByteBuffer getTagsByteBuffer() {
552       return this.tagBuff;
553     }
554 
555     @Override
556     public int getTagsPosition() {
557       return 0;
558     }
559 
560     @Override
561     public ByteBuffer getValueByteBuffer() {
562       return this.val;
563     }
564 
565     @Override
566     public int getValuePosition() {
567       return this.valOffset;
568     }
569 
570     @Override
571     public long heapSize() {
572       return FIXED_OVERHEAD + rowLength + famLength + qualLength + valLength + tagsLength;
573     }
574 
575     @Override
576     public String toString() {
577       String row = Bytes.toStringBinary(getRowArray(), getRowOffset(), getRowLength());
578       String family = Bytes.toStringBinary(getFamilyArray(), getFamilyOffset(), getFamilyLength());
579       String qualifier = Bytes.toStringBinary(getQualifierArray(), getQualifierOffset(),
580           getQualifierLength());
581       String timestamp = String.valueOf((getTimestamp()));
582       return row + "/" + family + (family != null && family.length() > 0 ? ":" : "") + qualifier
583           + "/" + timestamp + "/" + Type.codeToType(type);
584     }
585   }
586 }