View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.codec.prefixtree;
20  
21  import java.io.DataInputStream;
22  import java.io.DataOutputStream;
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  
26  import org.apache.hadoop.classification.InterfaceAudience;
27  import org.apache.hadoop.hbase.Cell;
28  import org.apache.hadoop.hbase.CellUtil;
29  import org.apache.hadoop.hbase.KeyValue;
30  import org.apache.hadoop.hbase.KeyValue.KVComparator;
31  import org.apache.hadoop.hbase.KeyValue.MetaComparator;
32  import org.apache.hadoop.hbase.KeyValue.RawBytesComparator;
33  import org.apache.hadoop.hbase.KeyValueUtil;
34  import org.apache.hadoop.hbase.codec.prefixtree.decode.DecoderFactory;
35  import org.apache.hadoop.hbase.codec.prefixtree.decode.PrefixTreeArraySearcher;
36  import org.apache.hadoop.hbase.codec.prefixtree.encode.EncoderFactory;
37  import org.apache.hadoop.hbase.codec.prefixtree.encode.PrefixTreeEncoder;
38  import org.apache.hadoop.hbase.codec.prefixtree.scanner.CellSearcher;
39  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
40  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
41  import org.apache.hadoop.hbase.io.encoding.EncodingState;
42  import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
43  import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultDecodingContext;
44  import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
45  import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
46  import org.apache.hadoop.hbase.io.hfile.BlockType;
47  import org.apache.hadoop.hbase.io.hfile.HFileContext;
48  import org.apache.hadoop.hbase.util.ByteBufferUtils;
49  import org.apache.hadoop.io.WritableUtils;
50  
51  /**
52   * This class is created via reflection in DataBlockEncoding enum. Update the enum if class name or
53   * package changes.
54   * <p/>
55   * PrefixTreeDataBlockEncoder implementation of DataBlockEncoder. This is the primary entry point
56   * for PrefixTree encoding and decoding. Encoding is delegated to instances of
57   * {@link PrefixTreeEncoder}, and decoding is delegated to instances of
58   * {@link org.apache.hadoop.hbase.codec.prefixtree.scanner.CellSearcher}. Encoder and decoder instances are
59   * created and recycled by static PtEncoderFactory and PtDecoderFactory.
60   */
61  @InterfaceAudience.Private
62  public class PrefixTreeCodec implements DataBlockEncoder{
63  
64    /**
65     * no-arg constructor for reflection
66     */
67    public PrefixTreeCodec() {
68    }
69  
70    @Override
71    public ByteBuffer decodeKeyValues(DataInputStream source, HFileBlockDecodingContext decodingCtx)
72        throws IOException {
73      return decodeKeyValues(source, 0, 0, decodingCtx);
74    }
75  
76  
77    /**
78     * I don't think this method is called during normal HBase operation, so efficiency is not
79     * important.
80     */
81    public ByteBuffer decodeKeyValues(DataInputStream source, int allocateHeaderLength,
82        int skipLastBytes, HFileBlockDecodingContext decodingCtx) throws IOException {
83      ByteBuffer sourceAsBuffer = ByteBufferUtils.drainInputStreamToBuffer(source);// waste
84      sourceAsBuffer.mark();
85      PrefixTreeBlockMeta blockMeta = new PrefixTreeBlockMeta(sourceAsBuffer);
86      sourceAsBuffer.rewind();
87      int numV1BytesWithHeader = allocateHeaderLength + blockMeta.getNumKeyValueBytes();
88      byte[] keyValueBytesWithHeader = new byte[numV1BytesWithHeader];
89      ByteBuffer result = ByteBuffer.wrap(keyValueBytesWithHeader);
90      result.rewind();
91      CellSearcher searcher = null;
92      try {
93        boolean includesMvcc = decodingCtx.getHFileContext().isIncludesMvcc();
94        searcher = DecoderFactory.checkOut(sourceAsBuffer, includesMvcc);
95        while (searcher.advance()) {
96          KeyValue currentCell = KeyValueUtil.copyToNewKeyValue(searcher.current());
97          // needs to be modified for DirectByteBuffers. no existing methods to
98          // write VLongs to byte[]
99          int offset = result.arrayOffset() + result.position();
100         System.arraycopy(currentCell.getBuffer(), currentCell.getOffset(), result.array(), offset,
101             currentCell.getLength());
102         int keyValueLength = KeyValueUtil.length(currentCell);
103         ByteBufferUtils.skip(result, keyValueLength);
104         offset += keyValueLength;
105         if (includesMvcc) {
106           ByteBufferUtils.writeVLong(result, currentCell.getMvccVersion());
107         }
108       }
109       result.position(result.limit());//make it appear as if we were appending
110       return result;
111     } finally {
112       DecoderFactory.checkIn(searcher);
113     }
114   }
115 
116 
117   @Override
118   public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
119     block.rewind();
120     PrefixTreeArraySearcher searcher = null;
121     try {
122       // should i includeMemstoreTS (second argument)?  i think PrefixKeyDeltaEncoder is, so i will
123       searcher = DecoderFactory.checkOut(block, true);
124       if (!searcher.positionAtFirstCell()) {
125         return null;
126       }
127       return KeyValueUtil.copyKeyToNewByteBuffer(searcher.current());
128     } finally {
129       DecoderFactory.checkIn(searcher);
130     }
131   }
132 
133   @Override
134   public HFileBlockEncodingContext newDataBlockEncodingContext(
135       DataBlockEncoding encoding, byte[] header, HFileContext meta) {
136     if(DataBlockEncoding.PREFIX_TREE != encoding){
137       //i'm not sure why encoding is in the interface.  Each encoder implementation should probably
138       //know it's encoding type
139       throw new IllegalArgumentException("only DataBlockEncoding.PREFIX_TREE supported");
140     }
141     return new HFileBlockDefaultEncodingContext(encoding, header, meta);
142   }
143 
144   @Override
145   public HFileBlockDecodingContext newDataBlockDecodingContext(HFileContext meta) {
146     return new HFileBlockDefaultDecodingContext(meta);
147   }
148 
149   /**
150    * Is this the correct handling of an illegal comparator?  How to prevent that from getting all
151    * the way to this point.
152    */
153   @Override
154   public EncodedSeeker createSeeker(KVComparator comparator, HFileBlockDecodingContext decodingCtx) {
155     if (comparator instanceof RawBytesComparator){
156       throw new IllegalArgumentException("comparator must be KeyValue.KeyComparator");
157     } else if (comparator instanceof MetaComparator){
158       throw new IllegalArgumentException("DataBlockEncoding.PREFIX_TREE not compatible with hbase:meta "
159           +"table");
160     }
161 
162     return new PrefixTreeSeeker(decodingCtx.getHFileContext().isIncludesMvcc());
163   }
164 
165   @Override
166   public int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out)
167       throws IOException {
168     PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState();
169     PrefixTreeEncoder builder = state.builder;
170     builder.write(cell);
171     int size = KeyValueUtil.length(cell);
172     if (encodingCtx.getHFileContext().isIncludesMvcc()) {
173       size += WritableUtils.getVIntSize(cell.getSequenceId());
174     }
175     return size;
176   }
177 
178   private static class PrefixTreeEncodingState extends EncodingState {
179     PrefixTreeEncoder builder = null;
180   }
181 
182   @Override
183   public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out)
184       throws IOException {
185     if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
186       throw new IOException(this.getClass().getName() + " only accepts "
187           + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context.");
188     }
189 
190     HFileBlockDefaultEncodingContext encodingCtx = 
191         (HFileBlockDefaultEncodingContext) blkEncodingCtx;
192     encodingCtx.prepareEncoding(out);
193 
194     PrefixTreeEncoder builder = EncoderFactory.checkOut(out, encodingCtx.getHFileContext()
195         .isIncludesMvcc());
196     PrefixTreeEncodingState state = new PrefixTreeEncodingState();
197     state.builder = builder;
198     blkEncodingCtx.setEncodingState(state);
199   }
200 
201   @Override
202   public void endBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out,
203       byte[] uncompressedBytesWithHeader) throws IOException {
204     PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState();
205     PrefixTreeEncoder builder = state.builder;
206     builder.flush();
207     EncoderFactory.checkIn(builder);
208     // do i need to check this, or will it always be DataBlockEncoding.PREFIX_TREE?
209     if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
210       encodingCtx.postEncoding(BlockType.ENCODED_DATA);
211     } else {
212       encodingCtx.postEncoding(BlockType.DATA);
213     }
214   }
215 }