1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.codec.prefixtree;
20
21 import java.io.DataInputStream;
22 import java.io.DataOutputStream;
23 import java.io.IOException;
24 import java.nio.ByteBuffer;
25
26 import org.apache.hadoop.hbase.classification.InterfaceAudience;
27 import org.apache.hadoop.hbase.Cell;
28 import org.apache.hadoop.hbase.KeyValue;
29 import org.apache.hadoop.hbase.KeyValue.KVComparator;
30 import org.apache.hadoop.hbase.KeyValue.MetaComparator;
31 import org.apache.hadoop.hbase.KeyValue.RawBytesComparator;
32 import org.apache.hadoop.hbase.KeyValueUtil;
33 import org.apache.hadoop.hbase.codec.prefixtree.decode.DecoderFactory;
34 import org.apache.hadoop.hbase.codec.prefixtree.decode.PrefixTreeArraySearcher;
35 import org.apache.hadoop.hbase.codec.prefixtree.encode.EncoderFactory;
36 import org.apache.hadoop.hbase.codec.prefixtree.encode.PrefixTreeEncoder;
37 import org.apache.hadoop.hbase.codec.prefixtree.scanner.CellSearcher;
38 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
39 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
40 import org.apache.hadoop.hbase.io.encoding.EncodingState;
41 import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
42 import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultDecodingContext;
43 import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
44 import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
45 import org.apache.hadoop.hbase.io.hfile.BlockType;
46 import org.apache.hadoop.hbase.io.hfile.HFileContext;
47 import org.apache.hadoop.hbase.util.ByteBufferUtils;
48 import org.apache.hadoop.io.WritableUtils;
49
50
51
52
53
54
55
56
57
58
59
60 @InterfaceAudience.Private
61 public class PrefixTreeCodec implements DataBlockEncoder{
62
63
64
65
66 public PrefixTreeCodec() {
67 }
68
69 @Override
70 public ByteBuffer decodeKeyValues(DataInputStream source, HFileBlockDecodingContext decodingCtx)
71 throws IOException {
72 return decodeKeyValues(source, 0, 0, decodingCtx);
73 }
74
75
76
77
78
79
80 public ByteBuffer decodeKeyValues(DataInputStream source, int allocateHeaderLength,
81 int skipLastBytes, HFileBlockDecodingContext decodingCtx) throws IOException {
82 ByteBuffer sourceAsBuffer = ByteBufferUtils.drainInputStreamToBuffer(source);
83 sourceAsBuffer.mark();
84 PrefixTreeBlockMeta blockMeta = new PrefixTreeBlockMeta(sourceAsBuffer);
85 sourceAsBuffer.rewind();
86 int numV1BytesWithHeader = allocateHeaderLength + blockMeta.getNumKeyValueBytes();
87 byte[] keyValueBytesWithHeader = new byte[numV1BytesWithHeader];
88 ByteBuffer result = ByteBuffer.wrap(keyValueBytesWithHeader);
89 result.rewind();
90 CellSearcher searcher = null;
91 try {
92 boolean includesMvcc = decodingCtx.getHFileContext().isIncludesMvcc();
93 searcher = DecoderFactory.checkOut(sourceAsBuffer, includesMvcc);
94 while (searcher.advance()) {
95 KeyValue currentCell = KeyValueUtil.copyToNewKeyValue(searcher.current());
96
97
98 int offset = result.arrayOffset() + result.position();
99 System.arraycopy(currentCell.getBuffer(), currentCell.getOffset(), result.array(), offset,
100 currentCell.getLength());
101 int keyValueLength = KeyValueUtil.length(currentCell);
102 ByteBufferUtils.skip(result, keyValueLength);
103 offset += keyValueLength;
104 if (includesMvcc) {
105 ByteBufferUtils.writeVLong(result, currentCell.getMvccVersion());
106 }
107 }
108 result.position(result.limit());
109 return result;
110 } finally {
111 DecoderFactory.checkIn(searcher);
112 }
113 }
114
115
116 @Override
117 public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
118 block.rewind();
119 PrefixTreeArraySearcher searcher = null;
120 try {
121
122 searcher = DecoderFactory.checkOut(block, true);
123 if (!searcher.positionAtFirstCell()) {
124 return null;
125 }
126 return KeyValueUtil.copyKeyToNewByteBuffer(searcher.current());
127 } finally {
128 DecoderFactory.checkIn(searcher);
129 }
130 }
131
132 @Override
133 public HFileBlockEncodingContext newDataBlockEncodingContext(
134 DataBlockEncoding encoding, byte[] header, HFileContext meta) {
135 if(DataBlockEncoding.PREFIX_TREE != encoding){
136
137
138 throw new IllegalArgumentException("only DataBlockEncoding.PREFIX_TREE supported");
139 }
140 return new HFileBlockDefaultEncodingContext(encoding, header, meta);
141 }
142
143 @Override
144 public HFileBlockDecodingContext newDataBlockDecodingContext(HFileContext meta) {
145 return new HFileBlockDefaultDecodingContext(meta);
146 }
147
148
149
150
151
152 @Override
153 public EncodedSeeker createSeeker(KVComparator comparator, HFileBlockDecodingContext decodingCtx) {
154 if (comparator instanceof RawBytesComparator){
155 throw new IllegalArgumentException("comparator must be KeyValue.KeyComparator");
156 } else if (comparator instanceof MetaComparator){
157 throw new IllegalArgumentException("DataBlockEncoding.PREFIX_TREE not compatible with hbase:meta "
158 +"table");
159 }
160
161 return new PrefixTreeSeeker(decodingCtx.getHFileContext().isIncludesMvcc());
162 }
163
164 @Override
165 public int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out)
166 throws IOException {
167 PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState();
168 PrefixTreeEncoder builder = state.builder;
169 builder.write(cell);
170 int size = KeyValueUtil.length(cell);
171 if (encodingCtx.getHFileContext().isIncludesMvcc()) {
172 size += WritableUtils.getVIntSize(cell.getSequenceId());
173 }
174 return size;
175 }
176
177 private static class PrefixTreeEncodingState extends EncodingState {
178 PrefixTreeEncoder builder = null;
179 }
180
181 @Override
182 public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out)
183 throws IOException {
184 if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
185 throw new IOException(this.getClass().getName() + " only accepts "
186 + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context.");
187 }
188
189 HFileBlockDefaultEncodingContext encodingCtx =
190 (HFileBlockDefaultEncodingContext) blkEncodingCtx;
191 encodingCtx.prepareEncoding(out);
192
193 PrefixTreeEncoder builder = EncoderFactory.checkOut(out, encodingCtx.getHFileContext()
194 .isIncludesMvcc());
195 PrefixTreeEncodingState state = new PrefixTreeEncodingState();
196 state.builder = builder;
197 blkEncodingCtx.setEncodingState(state);
198 }
199
200 @Override
201 public void endBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out,
202 byte[] uncompressedBytesWithHeader) throws IOException {
203 PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState();
204 PrefixTreeEncoder builder = state.builder;
205 builder.flush();
206 EncoderFactory.checkIn(builder);
207
208 if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
209 encodingCtx.postEncoding(BlockType.ENCODED_DATA);
210 } else {
211 encodingCtx.postEncoding(BlockType.DATA);
212 }
213 }
214 }