View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.codec.prefixtree.encode.row;
20  
21  import java.io.IOException;
22  import java.io.OutputStream;
23  import java.util.ArrayList;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeBlockMeta;
29  import org.apache.hadoop.hbase.codec.prefixtree.encode.PrefixTreeEncoder;
30  import org.apache.hadoop.hbase.codec.prefixtree.encode.tokenize.TokenizerNode;
31  import org.apache.hadoop.hbase.util.ByteRangeUtils;
32  import org.apache.hadoop.hbase.util.CollectionUtils;
33  import org.apache.hadoop.hbase.util.vint.UFIntTool;
34  import org.apache.hadoop.hbase.util.vint.UVIntTool;
35  
36  /**
37   * Serializes the fields comprising one node of the row trie, which can be a branch, nub, or leaf.
38   * Please see the write() method for the order in which data is written.
39   */
40  @InterfaceAudience.Private
41  public class RowNodeWriter{
42    protected static final Log LOG = LogFactory.getLog(RowNodeWriter.class);
43  
44    /********************* fields ******************************/
45  
46    protected PrefixTreeEncoder prefixTreeEncoder;
47    protected PrefixTreeBlockMeta blockMeta;
48    protected TokenizerNode tokenizerNode;
49  
50    protected int tokenWidth;
51    protected int fanOut;
52    protected int numCells;
53  
54    protected int width;
55  
56  
57    /*********************** construct *************************/
58  
59    public RowNodeWriter(PrefixTreeEncoder keyValueBuilder, TokenizerNode tokenizerNode) {
60      reconstruct(keyValueBuilder, tokenizerNode);
61    }
62  
63    public void reconstruct(PrefixTreeEncoder prefixTreeEncoder, TokenizerNode tokenizerNode) {
64      this.prefixTreeEncoder = prefixTreeEncoder;
65      reset(tokenizerNode);
66    }
67  
68    public void reset(TokenizerNode node) {
69      this.blockMeta = prefixTreeEncoder.getBlockMeta();// changes between blocks
70      this.tokenizerNode = node;
71      this.tokenWidth = 0;
72      this.fanOut = 0;
73      this.numCells = 0;
74      this.width = 0;
75      calculateOffsetsAndLengths();
76    }
77  
78  
79    /********************* methods ****************************/
80  
81    protected void calculateOffsetsAndLengths(){
82      tokenWidth = tokenizerNode.getTokenLength();
83      if(!tokenizerNode.isRoot()){
84        --tokenWidth;//root has no parent
85      }
86      fanOut = CollectionUtils.nullSafeSize(tokenizerNode.getChildren());
87      numCells = tokenizerNode.getNumOccurrences();
88    }
89  
90    public int calculateWidth(){
91      calculateWidthOverrideOffsetWidth(blockMeta.getNextNodeOffsetWidth());
92      return width;
93    }
94  
95    public int calculateWidthOverrideOffsetWidth(int offsetWidth){
96      width = 0;
97      width += UVIntTool.numBytes(tokenWidth);
98      width += tokenWidth;
99  
100     width += UVIntTool.numBytes(fanOut);
101     width += fanOut;
102 
103     width += UVIntTool.numBytes(numCells);
104 
105     if(tokenizerNode.hasOccurrences()){
106       int fixedBytesPerCell = blockMeta.getFamilyOffsetWidth()
107         + blockMeta.getQualifierOffsetWidth()
108         + blockMeta.getTagsOffsetWidth()
109         + blockMeta.getTimestampIndexWidth()
110         + blockMeta.getMvccVersionIndexWidth()
111         + blockMeta.getKeyValueTypeWidth()
112         + blockMeta.getValueOffsetWidth()
113         + blockMeta.getValueLengthWidth();
114       width += numCells * fixedBytesPerCell;
115     }
116 
117     if( ! tokenizerNode.isLeaf()){
118       width += fanOut * offsetWidth;
119     }
120 
121     return width;
122   }
123 
124 
125   /*********************** writing the compiled structure to the OutputStream ***************/
126 
127   public void write(OutputStream os) throws IOException{
128     //info about this row trie node
129     writeRowToken(os);
130     writeFan(os);
131     writeNumCells(os);
132 
133     //UFInt indexes and offsets for each cell in the row (if nub or leaf)
134     writeFamilyNodeOffsets(os);
135     writeQualifierNodeOffsets(os);
136     writeTagNodeOffsets(os);
137     writeTimestampIndexes(os);
138     writeMvccVersionIndexes(os);
139     writeCellTypes(os);
140     writeValueOffsets(os);
141     writeValueLengths(os);
142     //offsets to the children of this row trie node (if branch or nub)
143     writeNextRowTrieNodeOffsets(os);
144   }
145 
146 
147   /**
148    * Row node token, fan, and numCells. Written once at the beginning of each row node. These 3
149    * fields can reproduce all the row keys that compose the block.
150    */
151 
152   /**
153    * UVInt: tokenWidth
154    * bytes: token
155    */
156   protected void writeRowToken(OutputStream os) throws IOException {
157     UVIntTool.writeBytes(tokenWidth, os);
158     int tokenStartIndex = tokenizerNode.isRoot() ? 0 : 1;
159     ByteRangeUtils.write(os, tokenizerNode.getToken(), tokenStartIndex);
160   }
161 
162   /**
163    * UVInt: numFanBytes/fanOut
164    * bytes: each fan byte
165    */
166   public void writeFan(OutputStream os) throws IOException {
167     UVIntTool.writeBytes(fanOut, os);
168     if (fanOut <= 0) {
169       return;
170     }
171     ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
172     for (int i = 0; i < children.size(); ++i) {
173       TokenizerNode child = children.get(i);
174       os.write(child.getToken().get(0));// first byte of each child's token
175     }
176   }
177 
178   /**
179    * UVInt: numCells, the number of cells in this row which will be 0 for branch nodes
180    */
181   protected void writeNumCells(OutputStream os) throws IOException {
182     UVIntTool.writeBytes(numCells, os);
183   }
184 
185 
186   /**
187    * The following methods write data for each cell in the row, mostly consisting of indexes or
188    * offsets into the timestamp/column data structures that are written in the middle of the block.
189    * We use {@link UFIntTool} to encode these indexes/offsets to allow random access during a binary
190    * search of a particular column/timestamp combination.
191    * <p>
192    * Branch nodes will not have any data in these sections.
193    * </p>
194    */
195 
196   protected void writeFamilyNodeOffsets(OutputStream os) throws IOException {
197     if (blockMeta.getFamilyOffsetWidth() <= 0) {
198       return;
199     }
200     for (int i = 0; i < numCells; ++i) {
201       int cellInsertionIndex = PrefixTreeEncoder.MULITPLE_FAMILIES_POSSIBLE ? tokenizerNode
202           .getFirstInsertionIndex() + i : 0;
203       int sortedIndex = prefixTreeEncoder.getFamilySorter().getSortedIndexForInsertionId(
204         cellInsertionIndex);
205       int indexedFamilyOffset = prefixTreeEncoder.getFamilyWriter().getOutputArrayOffset(
206         sortedIndex);
207       UFIntTool.writeBytes(blockMeta.getFamilyOffsetWidth(), indexedFamilyOffset, os);
208     }
209   }
210 
211   protected void writeQualifierNodeOffsets(OutputStream os) throws IOException {
212     if (blockMeta.getQualifierOffsetWidth() <= 0) {
213       return;
214     }
215     for (int i = 0; i < numCells; ++i) {
216       int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
217       int sortedIndex = prefixTreeEncoder.getQualifierSorter().getSortedIndexForInsertionId(
218         cellInsertionIndex);
219       int indexedQualifierOffset = prefixTreeEncoder.getQualifierWriter().getOutputArrayOffset(
220         sortedIndex);
221       UFIntTool.writeBytes(blockMeta.getQualifierOffsetWidth(), indexedQualifierOffset, os);
222     }
223   }
224 
225   protected void writeTagNodeOffsets(OutputStream os) throws IOException {
226     if (blockMeta.getTagsOffsetWidth() <= 0) {
227       return;
228     }
229     for (int i = 0; i < numCells; ++i) {
230       int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
231       int sortedIndex = prefixTreeEncoder.getTagSorter().getSortedIndexForInsertionId(
232         cellInsertionIndex);
233       int indexedTagOffset = prefixTreeEncoder.getTagWriter().getOutputArrayOffset(
234         sortedIndex);
235       UFIntTool.writeBytes(blockMeta.getTagsOffsetWidth(), indexedTagOffset, os);
236     }
237   }
238 
239   protected void writeTimestampIndexes(OutputStream os) throws IOException {
240     if (blockMeta.getTimestampIndexWidth() <= 0) {
241       return;
242     }
243     for (int i = 0; i < numCells; ++i) {
244       int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
245       long timestamp = prefixTreeEncoder.getTimestamps()[cellInsertionIndex];
246       int timestampIndex = prefixTreeEncoder.getTimestampEncoder().getIndex(timestamp);
247       UFIntTool.writeBytes(blockMeta.getTimestampIndexWidth(), timestampIndex, os);
248     }
249   }
250 
251   protected void writeMvccVersionIndexes(OutputStream os) throws IOException {
252     if (blockMeta.getMvccVersionIndexWidth() <= 0) {
253       return;
254     }
255     for (int i = 0; i < numCells; ++i) {
256       int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
257       long mvccVersion = prefixTreeEncoder.getMvccVersions()[cellInsertionIndex];
258       int mvccVersionIndex = prefixTreeEncoder.getMvccVersionEncoder().getIndex(mvccVersion);
259       UFIntTool.writeBytes(blockMeta.getMvccVersionIndexWidth(), mvccVersionIndex, os);
260     }
261   }
262 
263   protected void writeCellTypes(OutputStream os) throws IOException {
264     if (blockMeta.isAllSameType()) {
265       return;
266     }
267     for (int i = 0; i < numCells; ++i) {
268       int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
269       os.write(prefixTreeEncoder.getTypeBytes()[cellInsertionIndex]);
270     }
271   }
272 
273   protected void writeValueOffsets(OutputStream os) throws IOException {
274     for (int i = 0; i < numCells; ++i) {
275       int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
276       long valueStartIndex = prefixTreeEncoder.getValueOffset(cellInsertionIndex);
277       UFIntTool.writeBytes(blockMeta.getValueOffsetWidth(), valueStartIndex, os);
278     }
279   }
280 
281   protected void writeValueLengths(OutputStream os) throws IOException {
282     for (int i = 0; i < numCells; ++i) {
283       int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
284       int valueLength = prefixTreeEncoder.getValueLength(cellInsertionIndex);
285       UFIntTool.writeBytes(blockMeta.getValueLengthWidth(), valueLength, os);
286     }
287   }
288 
289   /**
290    * If a branch or a nub, the last thing we append are the UFInt offsets to the child row nodes.
291    */
292   protected void writeNextRowTrieNodeOffsets(OutputStream os) throws IOException {
293     ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
294     for (int i = 0; i < children.size(); ++i) {
295       TokenizerNode child = children.get(i);
296       int distanceToChild = tokenizerNode.getNegativeIndex() - child.getNegativeIndex();
297       UFIntTool.writeBytes(blockMeta.getNextNodeOffsetWidth(), distanceToChild, os);
298     }
299   }
300 }