1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.codec.prefixtree.encode.row;
20
21 import java.io.IOException;
22 import java.io.OutputStream;
23 import java.util.ArrayList;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.classification.InterfaceAudience;
28 import org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeBlockMeta;
29 import org.apache.hadoop.hbase.codec.prefixtree.encode.PrefixTreeEncoder;
30 import org.apache.hadoop.hbase.codec.prefixtree.encode.tokenize.TokenizerNode;
31 import org.apache.hadoop.hbase.util.ByteRangeUtils;
32 import org.apache.hadoop.hbase.util.CollectionUtils;
33 import org.apache.hadoop.hbase.util.vint.UFIntTool;
34 import org.apache.hadoop.hbase.util.vint.UVIntTool;
35
36
37
38
39
40 @InterfaceAudience.Private
41 public class RowNodeWriter{
42 protected static final Log LOG = LogFactory.getLog(RowNodeWriter.class);
43
44
45
46 protected PrefixTreeEncoder prefixTreeEncoder;
47 protected PrefixTreeBlockMeta blockMeta;
48 protected TokenizerNode tokenizerNode;
49
50 protected int tokenWidth;
51 protected int fanOut;
52 protected int numCells;
53
54 protected int width;
55
56
57
58
59 public RowNodeWriter(PrefixTreeEncoder keyValueBuilder, TokenizerNode tokenizerNode) {
60 reconstruct(keyValueBuilder, tokenizerNode);
61 }
62
63 public void reconstruct(PrefixTreeEncoder prefixTreeEncoder, TokenizerNode tokenizerNode) {
64 this.prefixTreeEncoder = prefixTreeEncoder;
65 reset(tokenizerNode);
66 }
67
68 public void reset(TokenizerNode node) {
69 this.blockMeta = prefixTreeEncoder.getBlockMeta();
70 this.tokenizerNode = node;
71 this.tokenWidth = 0;
72 this.fanOut = 0;
73 this.numCells = 0;
74 this.width = 0;
75 calculateOffsetsAndLengths();
76 }
77
78
79
80
81 protected void calculateOffsetsAndLengths(){
82 tokenWidth = tokenizerNode.getTokenLength();
83 if(!tokenizerNode.isRoot()){
84 --tokenWidth;
85 }
86 fanOut = CollectionUtils.nullSafeSize(tokenizerNode.getChildren());
87 numCells = tokenizerNode.getNumOccurrences();
88 }
89
90 public int calculateWidth(){
91 calculateWidthOverrideOffsetWidth(blockMeta.getNextNodeOffsetWidth());
92 return width;
93 }
94
95 public int calculateWidthOverrideOffsetWidth(int offsetWidth){
96 width = 0;
97 width += UVIntTool.numBytes(tokenWidth);
98 width += tokenWidth;
99
100 width += UVIntTool.numBytes(fanOut);
101 width += fanOut;
102
103 width += UVIntTool.numBytes(numCells);
104
105 if(tokenizerNode.hasOccurrences()){
106 int fixedBytesPerCell = blockMeta.getFamilyOffsetWidth()
107 + blockMeta.getQualifierOffsetWidth()
108 + blockMeta.getTagsOffsetWidth()
109 + blockMeta.getTimestampIndexWidth()
110 + blockMeta.getMvccVersionIndexWidth()
111 + blockMeta.getKeyValueTypeWidth()
112 + blockMeta.getValueOffsetWidth()
113 + blockMeta.getValueLengthWidth();
114 width += numCells * fixedBytesPerCell;
115 }
116
117 if( ! tokenizerNode.isLeaf()){
118 width += fanOut * offsetWidth;
119 }
120
121 return width;
122 }
123
124
125
126
127 public void write(OutputStream os) throws IOException{
128
129 writeRowToken(os);
130 writeFan(os);
131 writeNumCells(os);
132
133
134 writeFamilyNodeOffsets(os);
135 writeQualifierNodeOffsets(os);
136 writeTagNodeOffsets(os);
137 writeTimestampIndexes(os);
138 writeMvccVersionIndexes(os);
139 writeCellTypes(os);
140 writeValueOffsets(os);
141 writeValueLengths(os);
142
143 writeNextRowTrieNodeOffsets(os);
144 }
145
146
147
148
149
150
151
152
153
154
155
156 protected void writeRowToken(OutputStream os) throws IOException {
157 UVIntTool.writeBytes(tokenWidth, os);
158 int tokenStartIndex = tokenizerNode.isRoot() ? 0 : 1;
159 ByteRangeUtils.write(os, tokenizerNode.getToken(), tokenStartIndex);
160 }
161
162
163
164
165
166 public void writeFan(OutputStream os) throws IOException {
167 UVIntTool.writeBytes(fanOut, os);
168 if (fanOut <= 0) {
169 return;
170 }
171 ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
172 for (int i = 0; i < children.size(); ++i) {
173 TokenizerNode child = children.get(i);
174 os.write(child.getToken().get(0));
175 }
176 }
177
178
179
180
181 protected void writeNumCells(OutputStream os) throws IOException {
182 UVIntTool.writeBytes(numCells, os);
183 }
184
185
186
187
188
189
190
191
192
193
194
195
196 protected void writeFamilyNodeOffsets(OutputStream os) throws IOException {
197 if (blockMeta.getFamilyOffsetWidth() <= 0) {
198 return;
199 }
200 for (int i = 0; i < numCells; ++i) {
201 int cellInsertionIndex = PrefixTreeEncoder.MULITPLE_FAMILIES_POSSIBLE ? tokenizerNode
202 .getFirstInsertionIndex() + i : 0;
203 int sortedIndex = prefixTreeEncoder.getFamilySorter().getSortedIndexForInsertionId(
204 cellInsertionIndex);
205 int indexedFamilyOffset = prefixTreeEncoder.getFamilyWriter().getOutputArrayOffset(
206 sortedIndex);
207 UFIntTool.writeBytes(blockMeta.getFamilyOffsetWidth(), indexedFamilyOffset, os);
208 }
209 }
210
211 protected void writeQualifierNodeOffsets(OutputStream os) throws IOException {
212 if (blockMeta.getQualifierOffsetWidth() <= 0) {
213 return;
214 }
215 for (int i = 0; i < numCells; ++i) {
216 int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
217 int sortedIndex = prefixTreeEncoder.getQualifierSorter().getSortedIndexForInsertionId(
218 cellInsertionIndex);
219 int indexedQualifierOffset = prefixTreeEncoder.getQualifierWriter().getOutputArrayOffset(
220 sortedIndex);
221 UFIntTool.writeBytes(blockMeta.getQualifierOffsetWidth(), indexedQualifierOffset, os);
222 }
223 }
224
225 protected void writeTagNodeOffsets(OutputStream os) throws IOException {
226 if (blockMeta.getTagsOffsetWidth() <= 0) {
227 return;
228 }
229 for (int i = 0; i < numCells; ++i) {
230 int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
231 int sortedIndex = prefixTreeEncoder.getTagSorter().getSortedIndexForInsertionId(
232 cellInsertionIndex);
233 int indexedTagOffset = prefixTreeEncoder.getTagWriter().getOutputArrayOffset(
234 sortedIndex);
235 UFIntTool.writeBytes(blockMeta.getTagsOffsetWidth(), indexedTagOffset, os);
236 }
237 }
238
239 protected void writeTimestampIndexes(OutputStream os) throws IOException {
240 if (blockMeta.getTimestampIndexWidth() <= 0) {
241 return;
242 }
243 for (int i = 0; i < numCells; ++i) {
244 int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
245 long timestamp = prefixTreeEncoder.getTimestamps()[cellInsertionIndex];
246 int timestampIndex = prefixTreeEncoder.getTimestampEncoder().getIndex(timestamp);
247 UFIntTool.writeBytes(blockMeta.getTimestampIndexWidth(), timestampIndex, os);
248 }
249 }
250
251 protected void writeMvccVersionIndexes(OutputStream os) throws IOException {
252 if (blockMeta.getMvccVersionIndexWidth() <= 0) {
253 return;
254 }
255 for (int i = 0; i < numCells; ++i) {
256 int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
257 long mvccVersion = prefixTreeEncoder.getMvccVersions()[cellInsertionIndex];
258 int mvccVersionIndex = prefixTreeEncoder.getMvccVersionEncoder().getIndex(mvccVersion);
259 UFIntTool.writeBytes(blockMeta.getMvccVersionIndexWidth(), mvccVersionIndex, os);
260 }
261 }
262
263 protected void writeCellTypes(OutputStream os) throws IOException {
264 if (blockMeta.isAllSameType()) {
265 return;
266 }
267 for (int i = 0; i < numCells; ++i) {
268 int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
269 os.write(prefixTreeEncoder.getTypeBytes()[cellInsertionIndex]);
270 }
271 }
272
273 protected void writeValueOffsets(OutputStream os) throws IOException {
274 for (int i = 0; i < numCells; ++i) {
275 int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
276 long valueStartIndex = prefixTreeEncoder.getValueOffset(cellInsertionIndex);
277 UFIntTool.writeBytes(blockMeta.getValueOffsetWidth(), valueStartIndex, os);
278 }
279 }
280
281 protected void writeValueLengths(OutputStream os) throws IOException {
282 for (int i = 0; i < numCells; ++i) {
283 int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
284 int valueLength = prefixTreeEncoder.getValueLength(cellInsertionIndex);
285 UFIntTool.writeBytes(blockMeta.getValueLengthWidth(), valueLength, os);
286 }
287 }
288
289
290
291
292 protected void writeNextRowTrieNodeOffsets(OutputStream os) throws IOException {
293 ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
294 for (int i = 0; i < children.size(); ++i) {
295 TokenizerNode child = children.get(i);
296 int distanceToChild = tokenizerNode.getNegativeIndex() - child.getNegativeIndex();
297 UFIntTool.writeBytes(blockMeta.getNextNodeOffsetWidth(), distanceToChild, os);
298 }
299 }
300 }