View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.codec.prefixtree;
20  
21  import java.nio.ByteBuffer;
22  
23  import org.apache.hadoop.hbase.ByteBufferedCell;
24  import org.apache.hadoop.hbase.Cell;
25  import org.apache.hadoop.hbase.CellComparator;
26  import org.apache.hadoop.hbase.CellUtil;
27  import org.apache.hadoop.hbase.KeyValue.Type;
28  import org.apache.hadoop.hbase.KeyValueUtil;
29  import org.apache.hadoop.hbase.SettableSequenceId;
30  import org.apache.hadoop.hbase.classification.InterfaceAudience;
31  import org.apache.hadoop.hbase.codec.prefixtree.decode.DecoderFactory;
32  import org.apache.hadoop.hbase.codec.prefixtree.decode.PrefixTreeArraySearcher;
33  import org.apache.hadoop.hbase.codec.prefixtree.scanner.CellScannerPosition;
34  import org.apache.hadoop.hbase.io.HeapSize;
35  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder.EncodedSeeker;
36  import org.apache.hadoop.hbase.nio.ByteBuff;
37  import org.apache.hadoop.hbase.util.ByteBufferUtils;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.apache.hadoop.hbase.util.ClassSize;
40  
41  /**
42   * These methods have the same definition as any implementation of the EncodedSeeker.
43   *
44   * In the future, the EncodedSeeker could be modified to work with the Cell interface directly.  It
45   * currently returns a new KeyValue object each time getKeyValue is called.  This is not horrible,
46   * but in order to create a new KeyValue object, we must first allocate a new byte[] and copy in
47   * the data from the PrefixTreeCell.  It is somewhat heavyweight right now.
48   */
49  @InterfaceAudience.Private
50  public class PrefixTreeSeeker implements EncodedSeeker {
51  
52    protected ByteBuffer block;
53    protected boolean includeMvccVersion;
54    protected PrefixTreeArraySearcher ptSearcher;
55  
56    public PrefixTreeSeeker(boolean includeMvccVersion) {
57      this.includeMvccVersion = includeMvccVersion;
58    }
59  
60    @Override
61    public void setCurrentBuffer(ByteBuff fullBlockBuffer) {
62      ptSearcher = DecoderFactory.checkOut(fullBlockBuffer, includeMvccVersion);
63      rewind();
64    }
65  
66    /**
67     * <p>
68     * Currently unused.
69     * </p>
70     * TODO performance leak. should reuse the searchers. hbase does not currently have a hook where
71     * this can be called
72     */
73    public void releaseCurrentSearcher(){
74      DecoderFactory.checkIn(ptSearcher);
75    }
76  
77  
78    @Override
79    public Cell getKey() {
80      return ptSearcher.current();
81    }
82  
83  
84    @Override
85    public ByteBuffer getValueShallowCopy() {
86      return CellUtil.getValueBufferShallowCopy(ptSearcher.current());
87    }
88  
89    /**
90     * currently must do deep copy into new array
91     */
92    @Override
93    public Cell getCell() {
94      // The PrefixTreecell is of type BytebufferedCell and the value part of the cell
95      // determines whether we are offheap cell or onheap cell.  All other parts of the cell-
96      // row, fam and col are all represented as onheap byte[]
97      ByteBufferedCell cell = (ByteBufferedCell)ptSearcher.current();
98      if (cell == null) {
99        return null;
100     }
101     // Use the ByteBuffered cell to see if the Cell is onheap or offheap
102     if (cell.getValueByteBuffer().hasArray()) {
103       return new OnheapPrefixTreeCell(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
104           cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
105           cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
106           cell.getValueArray(), cell.getValueOffset(), cell.getValueLength(), cell.getTagsArray(),
107           cell.getTagsOffset(), cell.getTagsLength(), cell.getTimestamp(), cell.getTypeByte(),
108           cell.getSequenceId());
109     } else {
110       return new OffheapPrefixTreeCell(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
111           cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
112           cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
113           cell.getValueByteBuffer(), cell.getValuePosition(), cell.getValueLength(),
114           cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength(), cell.getTimestamp(),
115           cell.getTypeByte(), cell.getSequenceId());
116     }
117   }
118 
119   /**
120    * <p>
121    * Currently unused.
122    * </p><p>
123    * A nice, lightweight reference, though the underlying cell is transient. This method may return
124    * the same reference to the backing PrefixTreeCell repeatedly, while other implementations may
125    * return a different reference for each Cell.
126    * </p>
127    * The goal will be to transition the upper layers of HBase, like Filters and KeyValueHeap, to
128    * use this method instead of the getKeyValue() methods above.
129    */
130   public Cell get() {
131     return ptSearcher.current();
132   }
133 
134   @Override
135   public void rewind() {
136     ptSearcher.positionAtFirstCell();
137   }
138 
139   @Override
140   public boolean next() {
141     return ptSearcher.advance();
142   }
143 
144   public boolean advance() {
145     return ptSearcher.advance();
146   }
147 
148 
149   private static final boolean USE_POSITION_BEFORE = false;
150 
151   /*
152    * Support both of these options since the underlying PrefixTree supports
153    * both. Possibly expand the EncodedSeeker to utilize them both.
154    */
155 
156   protected int seekToOrBeforeUsingPositionAtOrBefore(Cell kv, boolean seekBefore) {
157     // this does a deep copy of the key byte[] because the CellSearcher
158     // interface wants a Cell
159     CellScannerPosition position = ptSearcher.seekForwardToOrBefore(kv);
160 
161     if (CellScannerPosition.AT == position) {
162       if (seekBefore) {
163         ptSearcher.previous();
164         return 1;
165       }
166       return 0;
167     }
168 
169     return 1;
170   }
171 
172   protected int seekToOrBeforeUsingPositionAtOrAfter(Cell kv, boolean seekBefore) {
173     // should probably switch this to use the seekForwardToOrBefore method
174     CellScannerPosition position = ptSearcher.seekForwardToOrAfter(kv);
175 
176     if (CellScannerPosition.AT == position) {
177       if (seekBefore) {
178         ptSearcher.previous();
179         return 1;
180       }
181       return 0;
182 
183     }
184 
185     if (CellScannerPosition.AFTER == position) {
186       if (!ptSearcher.isBeforeFirst()) {
187         ptSearcher.previous();
188       }
189       return 1;
190     }
191 
192     if (position == CellScannerPosition.AFTER_LAST) {
193       if (seekBefore) {
194         ptSearcher.previous();
195       }
196       return 1;
197     }
198 
199     throw new RuntimeException("unexpected CellScannerPosition:" + position);
200   }
201 
202   @Override
203   public int seekToKeyInBlock(Cell key, boolean forceBeforeOnExactMatch) {
204     if (USE_POSITION_BEFORE) {
205       return seekToOrBeforeUsingPositionAtOrBefore(key, forceBeforeOnExactMatch);
206     } else {
207       return seekToOrBeforeUsingPositionAtOrAfter(key, forceBeforeOnExactMatch);
208     }
209   }
210 
211   @Override
212   public int compareKey(CellComparator comparator, Cell key) {
213     return comparator.compare(key,
214         ptSearcher.current());
215   }
216 
217   /**
218    * Cloned version of the PrefixTreeCell where except the value part, the rest
219    * of the key part is deep copied
220    *
221    */
222   private static class OnheapPrefixTreeCell implements Cell, SettableSequenceId, HeapSize {
223     private static final long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT
224         + (5 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_LONG) + (4 * Bytes.SIZEOF_INT)
225         + (Bytes.SIZEOF_SHORT) + (2 * Bytes.SIZEOF_BYTE) + (5 * ClassSize.ARRAY));
226     private byte[] row;
227     private short rowLength;
228     private byte[] fam;
229     private byte famLength;
230     private byte[] qual;
231     private int qualLength;
232     private byte[] val;
233     private int valOffset;
234     private int valLength;
235     private byte[] tag;
236     private int tagsLength;
237     private long ts;
238     private long seqId;
239     private byte type;
240 
241     public OnheapPrefixTreeCell(byte[] row, int rowOffset, short rowLength, byte[] fam,
242         int famOffset, byte famLength, byte[] qual, int qualOffset, int qualLength, byte[] val,
243         int valOffset, int valLength, byte[] tag, int tagOffset, int tagLength, long ts, byte type,
244         long seqId) {
245       this.row = new byte[rowLength];
246       System.arraycopy(row, rowOffset, this.row, 0, rowLength);
247       this.rowLength = rowLength;
248       this.fam = new byte[famLength];
249       System.arraycopy(fam, famOffset, this.fam, 0, famLength);
250       this.famLength = famLength;
251       this.qual = new byte[qualLength];
252       System.arraycopy(qual, qualOffset, this.qual, 0, qualLength);
253       this.qualLength = qualLength;
254       this.tag = new byte[tagLength];
255       System.arraycopy(tag, tagOffset, this.tag, 0, tagLength);
256       this.tagsLength = tagLength;
257       this.val = val;
258       this.valLength = valLength;
259       this.valOffset = valOffset;
260       this.ts = ts;
261       this.seqId = seqId;
262       this.type = type;
263     }
264 
265     @Override
266     public void setSequenceId(long seqId) {
267       this.seqId = seqId;
268     }
269 
270     @Override
271     public byte[] getRowArray() {
272       return this.row;
273     }
274 
275     @Override
276     public int getRowOffset() {
277       return 0;
278     }
279 
280     @Override
281     public short getRowLength() {
282       return this.rowLength;
283     }
284 
285     @Override
286     public byte[] getFamilyArray() {
287       return this.fam;
288     }
289 
290     @Override
291     public int getFamilyOffset() {
292       return 0;
293     }
294 
295     @Override
296     public byte getFamilyLength() {
297       return this.famLength;
298     }
299 
300     @Override
301     public byte[] getQualifierArray() {
302       return this.qual;
303     }
304 
305     @Override
306     public int getQualifierOffset() {
307       return 0;
308     }
309 
310     @Override
311     public int getQualifierLength() {
312       return this.qualLength;
313     }
314 
315     @Override
316     public long getTimestamp() {
317       return ts;
318     }
319 
320     @Override
321     public byte getTypeByte() {
322       return type;
323     }
324 
325     @Override
326     public long getSequenceId() {
327       return seqId;
328     }
329 
330     @Override
331     public byte[] getValueArray() {
332       return val;
333     }
334 
335     @Override
336     public int getValueOffset() {
337       return this.valOffset;
338     }
339 
340     @Override
341     public int getValueLength() {
342       return this.valLength;
343     }
344 
345     @Override
346     public byte[] getTagsArray() {
347       return this.tag;
348     }
349 
350     @Override
351     public int getTagsOffset() {
352       return 0;
353     }
354 
355     @Override
356     public int getTagsLength() {
357       return this.tagsLength;
358     }
359 
360     @Override
361     public String toString() {
362       String row = Bytes.toStringBinary(getRowArray(), getRowOffset(), getRowLength());
363       String family = Bytes.toStringBinary(getFamilyArray(), getFamilyOffset(), getFamilyLength());
364       String qualifier = Bytes.toStringBinary(getQualifierArray(), getQualifierOffset(),
365           getQualifierLength());
366       String timestamp = String.valueOf((getTimestamp()));
367       return row + "/" + family + (family != null && family.length() > 0 ? ":" : "") + qualifier
368           + "/" + timestamp + "/" + Type.codeToType(type);
369     }
370 
371     @Override
372     public long heapSize() {
373       return FIXED_OVERHEAD + rowLength + famLength + qualLength + valLength + tagsLength;
374     }
375   }
376 
377   private static class OffheapPrefixTreeCell extends ByteBufferedCell implements Cell,
378       SettableSequenceId, HeapSize {
379     private static final long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT
380         + (5 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_LONG) + (4 * Bytes.SIZEOF_INT)
381         + (Bytes.SIZEOF_SHORT) + (2 * Bytes.SIZEOF_BYTE) + (5 * ClassSize.BYTE_BUFFER));
382     private ByteBuffer rowBuff;
383     private short rowLength;
384     private ByteBuffer famBuff;
385     private byte famLength;
386     private ByteBuffer qualBuff;
387     private int qualLength;
388     private ByteBuffer val;
389     private int valOffset;
390     private int valLength;
391     private ByteBuffer tagBuff;
392     private int tagsLength;
393     private long ts;
394     private long seqId;
395     private byte type;
396     public OffheapPrefixTreeCell(byte[] row, int rowOffset, short rowLength, byte[] fam,
397         int famOffset, byte famLength, byte[] qual, int qualOffset, int qualLength, ByteBuffer val,
398         int valOffset, int valLength, byte[] tag, int tagOffset, int tagLength, long ts, byte type,
399         long seqId) {
400       byte[] tmpRow = new byte[rowLength];
401       System.arraycopy(row, rowOffset, tmpRow, 0, rowLength);
402       this.rowBuff = ByteBuffer.wrap(tmpRow);
403       this.rowLength = rowLength;
404       byte[] tmpFam = new byte[famLength];
405       System.arraycopy(fam, famOffset, tmpFam, 0, famLength);
406       this.famBuff = ByteBuffer.wrap(tmpFam);
407       this.famLength = famLength;
408       byte[] tmpQual = new byte[qualLength];
409       System.arraycopy(qual, qualOffset, tmpQual, 0, qualLength);
410       this.qualBuff = ByteBuffer.wrap(tmpQual);
411       this.qualLength = qualLength;
412       byte[] tmpTag = new byte[tagLength];
413       System.arraycopy(tag, tagOffset, tmpTag, 0, tagLength);
414       this.tagBuff = ByteBuffer.wrap(tmpTag);
415       this.tagsLength = tagLength;
416       this.val = val;
417       this.valLength = valLength;
418       this.valOffset = valOffset;
419       this.ts = ts;
420       this.seqId = seqId;
421       this.type = type;
422     }
423     
424     @Override
425     public void setSequenceId(long seqId) {
426       this.seqId = seqId;
427     }
428 
429     @Override
430     public byte[] getRowArray() {
431       return this.rowBuff.array();
432     }
433 
434     @Override
435     public int getRowOffset() {
436       return getRowPosition();
437     }
438 
439     @Override
440     public short getRowLength() {
441       return this.rowLength;
442     }
443 
444     @Override
445     public byte[] getFamilyArray() {
446       return this.famBuff.array();
447     }
448 
449     @Override
450     public int getFamilyOffset() {
451       return getFamilyPosition();
452     }
453 
454     @Override
455     public byte getFamilyLength() {
456       return this.famLength;
457     }
458 
459     @Override
460     public byte[] getQualifierArray() {
461       return this.qualBuff.array();
462     }
463 
464     @Override
465     public int getQualifierOffset() {
466       return getQualifierPosition();
467     }
468 
469     @Override
470     public int getQualifierLength() {
471       return this.qualLength;
472     }
473 
474     @Override
475     public long getTimestamp() {
476       return ts;
477     }
478 
479     @Override
480     public byte getTypeByte() {
481       return type;
482     }
483 
484     @Override
485     public long getSequenceId() {
486       return seqId;
487     }
488 
489     @Override
490     public byte[] getValueArray() {
491       byte[] tmpVal = new byte[valLength];
492       ByteBufferUtils.copyFromBufferToArray(tmpVal, val, valOffset, 0, valLength);
493       return tmpVal;
494     }
495 
496     @Override
497     public int getValueOffset() {
498       return 0;
499     }
500 
501     @Override
502     public int getValueLength() {
503       return this.valLength;
504     }
505 
506     @Override
507     public byte[] getTagsArray() {
508       return this.tagBuff.array();
509     }
510 
511     @Override
512     public int getTagsOffset() {
513       return getTagsPosition();
514     }
515 
516     @Override
517     public int getTagsLength() {
518       return this.tagsLength;
519     }
520     
521     @Override
522     public ByteBuffer getRowByteBuffer() {
523       return this.rowBuff;
524     }
525     
526     @Override
527     public int getRowPosition() {
528       return 0;
529     }
530     
531     @Override
532     public ByteBuffer getFamilyByteBuffer() {
533       return this.famBuff;
534     }
535     
536     @Override
537     public int getFamilyPosition() {
538       return 0;
539     }
540     
541     @Override
542     public ByteBuffer getQualifierByteBuffer() {
543       return this.qualBuff;
544     }
545 
546     @Override
547     public int getQualifierPosition() {
548       return 0;
549     }
550 
551     @Override
552     public ByteBuffer getTagsByteBuffer() {
553       return this.tagBuff;
554     }
555 
556     @Override
557     public int getTagsPosition() {
558       return 0;
559     }
560 
561     @Override
562     public ByteBuffer getValueByteBuffer() {
563       return this.val;
564     }
565 
566     @Override
567     public int getValuePosition() {
568       return this.valOffset;
569     }
570 
571     @Override
572     public long heapSize() {
573       return FIXED_OVERHEAD + rowLength + famLength + qualLength + valLength + tagsLength;
574     }
575 
576     @Override
577     public String toString() {
578       String row = Bytes.toStringBinary(getRowArray(), getRowOffset(), getRowLength());
579       String family = Bytes.toStringBinary(getFamilyArray(), getFamilyOffset(), getFamilyLength());
580       String qualifier = Bytes.toStringBinary(getQualifierArray(), getQualifierOffset(),
581           getQualifierLength());
582       String timestamp = String.valueOf((getTimestamp()));
583       return row + "/" + family + (family != null && family.length() > 0 ? ":" : "") + qualifier
584           + "/" + timestamp + "/" + Type.codeToType(type);
585     }
586   }
587 }