View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.codec.prefixtree;
20  
21  import java.io.DataInputStream;
22  import java.io.DataOutputStream;
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  
26  import org.apache.hadoop.classification.InterfaceAudience;
27  import org.apache.hadoop.hbase.KeyValue;
28  import org.apache.hadoop.hbase.KeyValue.KVComparator;
29  import org.apache.hadoop.hbase.KeyValue.MetaComparator;
30  import org.apache.hadoop.hbase.KeyValue.RawBytesComparator;
31  import org.apache.hadoop.hbase.KeyValueUtil;
32  import org.apache.hadoop.hbase.codec.prefixtree.decode.DecoderFactory;
33  import org.apache.hadoop.hbase.codec.prefixtree.decode.PrefixTreeArraySearcher;
34  import org.apache.hadoop.hbase.codec.prefixtree.encode.EncoderFactory;
35  import org.apache.hadoop.hbase.codec.prefixtree.encode.PrefixTreeEncoder;
36  import org.apache.hadoop.hbase.codec.prefixtree.scanner.CellSearcher;
37  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
38  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
39  import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
40  import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultDecodingContext;
41  import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
42  import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
43  import org.apache.hadoop.hbase.io.hfile.BlockType;
44  import org.apache.hadoop.hbase.io.hfile.HFileContext;
45  import org.apache.hadoop.hbase.util.ByteBufferUtils;
46  
47  /**
48   * This class is created via reflection in DataBlockEncoding enum. Update the enum if class name or
49   * package changes.
50   * <p/>
51   * PrefixTreeDataBlockEncoder implementation of DataBlockEncoder. This is the primary entry point
52   * for PrefixTree encoding and decoding. Encoding is delegated to instances of
53   * {@link PrefixTreeEncoder}, and decoding is delegated to instances of
54   * {@link org.apache.hadoop.hbase.codec.prefixtree.scanner.CellSearcher}. Encoder and decoder instances are
55   * created and recycled by static PtEncoderFactory and PtDecoderFactory.
56   */
57  @InterfaceAudience.Private
58  public class PrefixTreeCodec implements DataBlockEncoder{
59  
60    /**
61     * no-arg constructor for reflection
62     */
63    public PrefixTreeCodec() {
64    }
65  
66    /**
67     * Copied from BufferedDataBlockEncoder. Almost definitely can be improved, but i'm not familiar
68     * enough with the concept of the HFileBlockEncodingContext.
69     */
70    @Override
71    public void encodeKeyValues(ByteBuffer in,
72        HFileBlockEncodingContext blkEncodingCtx) throws IOException {
73      if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
74        throw new IOException(this.getClass().getName() + " only accepts "
75            + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context.");
76      }
77  
78      HFileBlockDefaultEncodingContext encodingCtx
79          = (HFileBlockDefaultEncodingContext) blkEncodingCtx;
80      encodingCtx.prepareEncoding();
81      DataOutputStream dataOut = encodingCtx.getOutputStreamForEncoder();
82      internalEncodeKeyValues(dataOut, in, encodingCtx.getHFileContext().isIncludesMvcc(),
83          encodingCtx.getHFileContext().isIncludesTags());
84  
85      //do i need to check this, or will it always be DataBlockEncoding.PREFIX_TREE?
86      if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
87        encodingCtx.postEncoding(BlockType.ENCODED_DATA);
88      } else {
89        encodingCtx.postEncoding(BlockType.DATA);
90      }
91    }
92  
93    private void internalEncodeKeyValues(DataOutputStream encodedOutputStream,
94        ByteBuffer rawKeyValues, boolean includesMvccVersion, boolean includesTag) throws IOException {
95      rawKeyValues.rewind();
96      PrefixTreeEncoder builder = EncoderFactory.checkOut(encodedOutputStream, includesMvccVersion);
97  
98      try {
99        KeyValue kv;
100       while ((kv = KeyValueUtil.nextShallowCopy(rawKeyValues, includesMvccVersion, includesTag)) != null) {
101         builder.write(kv);
102       }
103       builder.flush();
104     } finally {
105       EncoderFactory.checkIn(builder);
106     }
107   }
108 
109 
110   @Override
111   public ByteBuffer decodeKeyValues(DataInputStream source, HFileBlockDecodingContext decodingCtx)
112       throws IOException {
113     return decodeKeyValues(source, 0, 0, decodingCtx);
114   }
115 
116 
117   /**
118    * I don't think this method is called during normal HBase operation, so efficiency is not
119    * important.
120    */
121   public ByteBuffer decodeKeyValues(DataInputStream source, int allocateHeaderLength,
122       int skipLastBytes, HFileBlockDecodingContext decodingCtx) throws IOException {
123     ByteBuffer sourceAsBuffer = ByteBufferUtils.drainInputStreamToBuffer(source);// waste
124     sourceAsBuffer.mark();
125     PrefixTreeBlockMeta blockMeta = new PrefixTreeBlockMeta(sourceAsBuffer);
126     sourceAsBuffer.rewind();
127     int numV1BytesWithHeader = allocateHeaderLength + blockMeta.getNumKeyValueBytes();
128     byte[] keyValueBytesWithHeader = new byte[numV1BytesWithHeader];
129     ByteBuffer result = ByteBuffer.wrap(keyValueBytesWithHeader);
130     result.rewind();
131     CellSearcher searcher = null;
132     try {
133       boolean includesMvcc = decodingCtx.getHFileContext().isIncludesMvcc();
134       searcher = DecoderFactory.checkOut(sourceAsBuffer, includesMvcc);
135       while (searcher.advance()) {
136         KeyValue currentCell = KeyValueUtil.copyToNewKeyValue(searcher.current());
137         // needs to be modified for DirectByteBuffers. no existing methods to
138         // write VLongs to byte[]
139         int offset = result.arrayOffset() + result.position();
140         System.arraycopy(currentCell.getBuffer(), currentCell.getOffset(), result.array(), offset,
141             currentCell.getLength());
142         int keyValueLength = KeyValueUtil.length(currentCell);
143         ByteBufferUtils.skip(result, keyValueLength);
144         offset += keyValueLength;
145         if (includesMvcc) {
146           ByteBufferUtils.writeVLong(result, currentCell.getMvccVersion());
147         }
148       }
149       result.position(result.limit());//make it appear as if we were appending
150       return result;
151     } finally {
152       DecoderFactory.checkIn(searcher);
153     }
154   }
155 
156 
157   @Override
158   public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
159     block.rewind();
160     PrefixTreeArraySearcher searcher = null;
161     try {
162       // should i includeMemstoreTS (second argument)?  i think PrefixKeyDeltaEncoder is, so i will
163       searcher = DecoderFactory.checkOut(block, true);
164       if (!searcher.positionAtFirstCell()) {
165         return null;
166       }
167       return KeyValueUtil.copyKeyToNewByteBuffer(searcher.current());
168     } finally {
169       DecoderFactory.checkIn(searcher);
170     }
171   }
172 
173   @Override
174   public HFileBlockEncodingContext newDataBlockEncodingContext(
175       DataBlockEncoding encoding, byte[] header, HFileContext meta) {
176     if(DataBlockEncoding.PREFIX_TREE != encoding){
177       //i'm not sure why encoding is in the interface.  Each encoder implementation should probably
178       //know it's encoding type
179       throw new IllegalArgumentException("only DataBlockEncoding.PREFIX_TREE supported");
180     }
181     return new HFileBlockDefaultEncodingContext(encoding, header, meta);
182   }
183 
184   @Override
185   public HFileBlockDecodingContext newDataBlockDecodingContext(HFileContext meta) {
186     return new HFileBlockDefaultDecodingContext(meta);
187   }
188 
189   /**
190    * Is this the correct handling of an illegal comparator?  How to prevent that from getting all
191    * the way to this point.
192    */
193   @Override
194   public EncodedSeeker createSeeker(KVComparator comparator, HFileBlockDecodingContext decodingCtx) {
195     if (comparator instanceof RawBytesComparator){
196       throw new IllegalArgumentException("comparator must be KeyValue.KeyComparator");
197     } else if (comparator instanceof MetaComparator){
198       throw new IllegalArgumentException("DataBlockEncoding.PREFIX_TREE not compatible with hbase:meta "
199           +"table");
200     }
201 
202     return new PrefixTreeSeeker(decodingCtx.getHFileContext().isIncludesMvcc());
203   }
204 
205 }