View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.codec.prefixtree;
20  
21  import java.io.DataInputStream;
22  import java.io.DataOutputStream;
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.hbase.Cell;
28  import org.apache.hadoop.hbase.CellComparator;
29  import org.apache.hadoop.hbase.KeyValue;
30  import org.apache.hadoop.hbase.CellComparator.MetaCellComparator;
31  import org.apache.hadoop.hbase.KeyValueUtil;
32  import org.apache.hadoop.hbase.codec.prefixtree.decode.DecoderFactory;
33  import org.apache.hadoop.hbase.codec.prefixtree.decode.PrefixTreeArraySearcher;
34  import org.apache.hadoop.hbase.codec.prefixtree.encode.EncoderFactory;
35  import org.apache.hadoop.hbase.codec.prefixtree.encode.PrefixTreeEncoder;
36  import org.apache.hadoop.hbase.codec.prefixtree.scanner.CellSearcher;
37  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
38  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
39  import org.apache.hadoop.hbase.io.encoding.EncodingState;
40  import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
41  import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultDecodingContext;
42  import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
43  import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
44  import org.apache.hadoop.hbase.io.hfile.BlockType;
45  import org.apache.hadoop.hbase.io.hfile.HFileContext;
46  import org.apache.hadoop.hbase.nio.ByteBuff;
47  import org.apache.hadoop.hbase.nio.SingleByteBuff;
48  import org.apache.hadoop.hbase.util.ByteBufferUtils;
49  import org.apache.hadoop.io.WritableUtils;
50  
51  /**
52   * <p>
53   * This class is created via reflection in DataBlockEncoding enum. Update the enum if class name or
54   * package changes.
55   * </p>
56   * PrefixTreeDataBlockEncoder implementation of DataBlockEncoder. This is the primary entry point
57   * for PrefixTree encoding and decoding. Encoding is delegated to instances of
58   * {@link PrefixTreeEncoder}, and decoding is delegated to instances of
59   * {@link org.apache.hadoop.hbase.codec.prefixtree.scanner.CellSearcher}.
60   * Encoder and decoder instances are
61   * created and recycled by static PtEncoderFactory and PtDecoderFactory.
62   */
63  @InterfaceAudience.Private
64  public class PrefixTreeCodec implements DataBlockEncoder {
65  
66    /**
67     * no-arg constructor for reflection
68     */
69    public PrefixTreeCodec() {
70    }
71  
72    @Override
73    public ByteBuffer decodeKeyValues(DataInputStream source, HFileBlockDecodingContext decodingCtx)
74        throws IOException {
75      return decodeKeyValues(source, 0, 0, decodingCtx);
76    }
77  
78  
79    /**
80     * I don't think this method is called during normal HBase operation, so efficiency is not
81     * important.
82     */
83    public ByteBuffer decodeKeyValues(DataInputStream source, int allocateHeaderLength,
84        int skipLastBytes, HFileBlockDecodingContext decodingCtx) throws IOException {
85      ByteBuffer sourceAsBuffer = ByteBufferUtils.drainInputStreamToBuffer(source);// waste
86      sourceAsBuffer.mark();
87      PrefixTreeBlockMeta blockMeta = new PrefixTreeBlockMeta(new SingleByteBuff(sourceAsBuffer));
88      sourceAsBuffer.rewind();
89      int numV1BytesWithHeader = allocateHeaderLength + blockMeta.getNumKeyValueBytes();
90      byte[] keyValueBytesWithHeader = new byte[numV1BytesWithHeader];
91      ByteBuffer result = ByteBuffer.wrap(keyValueBytesWithHeader);
92      result.rewind();
93      CellSearcher searcher = null;
94      try {
95        boolean includesMvcc = decodingCtx.getHFileContext().isIncludesMvcc();
96        searcher = DecoderFactory.checkOut(new SingleByteBuff(sourceAsBuffer), includesMvcc);
97        while (searcher.advance()) {
98          KeyValue currentCell = KeyValueUtil.copyToNewKeyValue(searcher.current());
99          // needs to be modified for DirectByteBuffers. no existing methods to
100         // write VLongs to byte[]
101         int offset = result.arrayOffset() + result.position();
102         System.arraycopy(currentCell.getBuffer(), currentCell.getOffset(), result.array(), offset,
103             currentCell.getLength());
104         int keyValueLength = KeyValueUtil.length(currentCell);
105         ByteBufferUtils.skip(result, keyValueLength);
106         offset += keyValueLength;
107         if (includesMvcc) {
108           ByteBufferUtils.writeVLong(result, currentCell.getSequenceId());
109         }
110       }
111       result.position(result.limit());//make it appear as if we were appending
112       return result;
113     } finally {
114       DecoderFactory.checkIn(searcher);
115     }
116   }
117 
118 
119   @Override
120   public Cell getFirstKeyCellInBlock(ByteBuff block) {
121     block.rewind();
122     PrefixTreeArraySearcher searcher = null;
123     try {
124       // should i includeMemstoreTS (second argument)?  i think PrefixKeyDeltaEncoder is, so i will
125       searcher = DecoderFactory.checkOut(block, true);
126       if (!searcher.positionAtFirstCell()) {
127         return null;
128       }
129       return searcher.current();
130     } finally {
131       DecoderFactory.checkIn(searcher);
132     }
133   }
134 
135   @Override
136   public HFileBlockEncodingContext newDataBlockEncodingContext(
137       DataBlockEncoding encoding, byte[] header, HFileContext meta) {
138     if(DataBlockEncoding.PREFIX_TREE != encoding){
139       //i'm not sure why encoding is in the interface.  Each encoder implementation should probably
140       //know it's encoding type
141       throw new IllegalArgumentException("only DataBlockEncoding.PREFIX_TREE supported");
142     }
143     return new HFileBlockDefaultEncodingContext(encoding, header, meta);
144   }
145 
146   @Override
147   public HFileBlockDecodingContext newDataBlockDecodingContext(HFileContext meta) {
148     return new HFileBlockDefaultDecodingContext(meta);
149   }
150 
151   /**
152    * Is this the correct handling of an illegal comparator?  How to prevent that from getting all
153    * the way to this point.
154    */
155   @Override
156   public EncodedSeeker createSeeker(CellComparator comparator,
157       HFileBlockDecodingContext decodingCtx) {
158     if (comparator instanceof MetaCellComparator) {
159       throw new IllegalArgumentException(
160           "DataBlockEncoding.PREFIX_TREE not compatible with hbase:meta " + "table");
161     }
162 
163     return new PrefixTreeSeeker(decodingCtx.getHFileContext().isIncludesMvcc());
164   }
165 
166   @Override
167   public int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out)
168       throws IOException {
169     PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState();
170     PrefixTreeEncoder builder = state.builder;
171     builder.write(cell);
172     int size = KeyValueUtil.length(cell);
173     if (encodingCtx.getHFileContext().isIncludesMvcc()) {
174       size += WritableUtils.getVIntSize(cell.getSequenceId());
175     }
176     return size;
177   }
178 
179   private static class PrefixTreeEncodingState extends EncodingState {
180     PrefixTreeEncoder builder = null;
181   }
182 
183   @Override
184   public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out)
185       throws IOException {
186     if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
187       throw new IOException(this.getClass().getName() + " only accepts "
188           + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context.");
189     }
190 
191     HFileBlockDefaultEncodingContext encodingCtx = 
192         (HFileBlockDefaultEncodingContext) blkEncodingCtx;
193     encodingCtx.prepareEncoding(out);
194 
195     PrefixTreeEncoder builder = EncoderFactory.checkOut(out, encodingCtx.getHFileContext()
196         .isIncludesMvcc());
197     PrefixTreeEncodingState state = new PrefixTreeEncodingState();
198     state.builder = builder;
199     blkEncodingCtx.setEncodingState(state);
200   }
201 
202   @Override
203   public void endBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out,
204       byte[] uncompressedBytesWithHeader) throws IOException {
205     PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState();
206     PrefixTreeEncoder builder = state.builder;
207     builder.flush();
208     EncoderFactory.checkIn(builder);
209     // do i need to check this, or will it always be DataBlockEncoding.PREFIX_TREE?
210     if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
211       encodingCtx.postEncoding(BlockType.ENCODED_DATA);
212     } else {
213       encodingCtx.postEncoding(BlockType.DATA);
214     }
215   }
216 }