View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.codec.prefixtree;
20  
21  import java.io.DataInputStream;
22  import java.io.DataOutputStream;
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  
26  import org.apache.hadoop.classification.InterfaceAudience;
27  import org.apache.hadoop.hbase.KeyValue;
28  import org.apache.hadoop.hbase.KeyValue.KVComparator;
29  import org.apache.hadoop.hbase.KeyValue.MetaComparator;
30  import org.apache.hadoop.hbase.KeyValue.RawBytesComparator;
31  import org.apache.hadoop.hbase.KeyValueUtil;
32  import org.apache.hadoop.hbase.codec.prefixtree.decode.DecoderFactory;
33  import org.apache.hadoop.hbase.codec.prefixtree.decode.PrefixTreeArraySearcher;
34  import org.apache.hadoop.hbase.codec.prefixtree.encode.EncoderFactory;
35  import org.apache.hadoop.hbase.codec.prefixtree.encode.PrefixTreeEncoder;
36  import org.apache.hadoop.hbase.codec.prefixtree.scanner.CellSearcher;
37  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
38  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
39  import org.apache.hadoop.hbase.io.encoding.EncodingState;
40  import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
41  import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultDecodingContext;
42  import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
43  import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
44  import org.apache.hadoop.hbase.io.hfile.BlockType;
45  import org.apache.hadoop.hbase.io.hfile.HFileContext;
46  import org.apache.hadoop.hbase.util.ByteBufferUtils;
47  import org.apache.hadoop.io.WritableUtils;
48  
49  /**
50   * This class is created via reflection in DataBlockEncoding enum. Update the enum if class name or
51   * package changes.
52   * <p/>
53   * PrefixTreeDataBlockEncoder implementation of DataBlockEncoder. This is the primary entry point
54   * for PrefixTree encoding and decoding. Encoding is delegated to instances of
55   * {@link PrefixTreeEncoder}, and decoding is delegated to instances of
56   * {@link org.apache.hadoop.hbase.codec.prefixtree.scanner.CellSearcher}. Encoder and decoder instances are
57   * created and recycled by static PtEncoderFactory and PtDecoderFactory.
58   */
59  @InterfaceAudience.Private
60  public class PrefixTreeCodec implements DataBlockEncoder{
61  
62    /**
63     * no-arg constructor for reflection
64     */
65    public PrefixTreeCodec() {
66    }
67  
68    @Override
69    public ByteBuffer decodeKeyValues(DataInputStream source, HFileBlockDecodingContext decodingCtx)
70        throws IOException {
71      return decodeKeyValues(source, 0, 0, decodingCtx);
72    }
73  
74  
75    /**
76     * I don't think this method is called during normal HBase operation, so efficiency is not
77     * important.
78     */
79    public ByteBuffer decodeKeyValues(DataInputStream source, int allocateHeaderLength,
80        int skipLastBytes, HFileBlockDecodingContext decodingCtx) throws IOException {
81      ByteBuffer sourceAsBuffer = ByteBufferUtils.drainInputStreamToBuffer(source);// waste
82      sourceAsBuffer.mark();
83      PrefixTreeBlockMeta blockMeta = new PrefixTreeBlockMeta(sourceAsBuffer);
84      sourceAsBuffer.rewind();
85      int numV1BytesWithHeader = allocateHeaderLength + blockMeta.getNumKeyValueBytes();
86      byte[] keyValueBytesWithHeader = new byte[numV1BytesWithHeader];
87      ByteBuffer result = ByteBuffer.wrap(keyValueBytesWithHeader);
88      result.rewind();
89      CellSearcher searcher = null;
90      try {
91        boolean includesMvcc = decodingCtx.getHFileContext().isIncludesMvcc();
92        searcher = DecoderFactory.checkOut(sourceAsBuffer, includesMvcc);
93        while (searcher.advance()) {
94          KeyValue currentCell = KeyValueUtil.copyToNewKeyValue(searcher.current());
95          // needs to be modified for DirectByteBuffers. no existing methods to
96          // write VLongs to byte[]
97          int offset = result.arrayOffset() + result.position();
98          System.arraycopy(currentCell.getBuffer(), currentCell.getOffset(), result.array(), offset,
99              currentCell.getLength());
100         int keyValueLength = KeyValueUtil.length(currentCell);
101         ByteBufferUtils.skip(result, keyValueLength);
102         offset += keyValueLength;
103         if (includesMvcc) {
104           ByteBufferUtils.writeVLong(result, currentCell.getMvccVersion());
105         }
106       }
107       result.position(result.limit());//make it appear as if we were appending
108       return result;
109     } finally {
110       DecoderFactory.checkIn(searcher);
111     }
112   }
113 
114 
115   @Override
116   public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
117     block.rewind();
118     PrefixTreeArraySearcher searcher = null;
119     try {
120       // should i includeMemstoreTS (second argument)?  i think PrefixKeyDeltaEncoder is, so i will
121       searcher = DecoderFactory.checkOut(block, true);
122       if (!searcher.positionAtFirstCell()) {
123         return null;
124       }
125       return KeyValueUtil.copyKeyToNewByteBuffer(searcher.current());
126     } finally {
127       DecoderFactory.checkIn(searcher);
128     }
129   }
130 
131   @Override
132   public HFileBlockEncodingContext newDataBlockEncodingContext(
133       DataBlockEncoding encoding, byte[] header, HFileContext meta) {
134     if(DataBlockEncoding.PREFIX_TREE != encoding){
135       //i'm not sure why encoding is in the interface.  Each encoder implementation should probably
136       //know it's encoding type
137       throw new IllegalArgumentException("only DataBlockEncoding.PREFIX_TREE supported");
138     }
139     return new HFileBlockDefaultEncodingContext(encoding, header, meta);
140   }
141 
142   @Override
143   public HFileBlockDecodingContext newDataBlockDecodingContext(HFileContext meta) {
144     return new HFileBlockDefaultDecodingContext(meta);
145   }
146 
147   /**
148    * Is this the correct handling of an illegal comparator?  How to prevent that from getting all
149    * the way to this point.
150    */
151   @Override
152   public EncodedSeeker createSeeker(KVComparator comparator, HFileBlockDecodingContext decodingCtx) {
153     if (comparator instanceof RawBytesComparator){
154       throw new IllegalArgumentException("comparator must be KeyValue.KeyComparator");
155     } else if (comparator instanceof MetaComparator){
156       throw new IllegalArgumentException("DataBlockEncoding.PREFIX_TREE not compatible with hbase:meta "
157           +"table");
158     }
159 
160     return new PrefixTreeSeeker(decodingCtx.getHFileContext().isIncludesMvcc());
161   }
162 
163   @Override
164   public int encode(KeyValue kv, HFileBlockEncodingContext encodingCtx, DataOutputStream out)
165       throws IOException {
166     PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState();
167     PrefixTreeEncoder builder = state.builder;
168     builder.write(kv);
169     int size = kv.getLength();
170     if (encodingCtx.getHFileContext().isIncludesMvcc()) {
171       size += WritableUtils.getVIntSize(kv.getMvccVersion());
172     }
173     return size;
174   }
175 
176   private static class PrefixTreeEncodingState extends EncodingState {
177     PrefixTreeEncoder builder = null;
178   }
179 
180   @Override
181   public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out)
182       throws IOException {
183     if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
184       throw new IOException(this.getClass().getName() + " only accepts "
185           + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context.");
186     }
187 
188     HFileBlockDefaultEncodingContext encodingCtx = 
189         (HFileBlockDefaultEncodingContext) blkEncodingCtx;
190     encodingCtx.prepareEncoding(out);
191 
192     PrefixTreeEncoder builder = EncoderFactory.checkOut(out, encodingCtx.getHFileContext()
193         .isIncludesMvcc());
194     PrefixTreeEncodingState state = new PrefixTreeEncodingState();
195     state.builder = builder;
196     blkEncodingCtx.setEncodingState(state);
197   }
198 
199   @Override
200   public void endBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out,
201       byte[] uncompressedBytesWithHeader) throws IOException {
202     PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState();
203     PrefixTreeEncoder builder = state.builder;
204     builder.flush();
205     EncoderFactory.checkIn(builder);
206     // do i need to check this, or will it always be DataBlockEncoding.PREFIX_TREE?
207     if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
208       encodingCtx.postEncoding(BlockType.ENCODED_DATA);
209     } else {
210       encodingCtx.postEncoding(BlockType.DATA);
211     }
212   }
213 }