View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.thrift2;
20  
21  import static org.apache.hadoop.hbase.util.Bytes.getBytes;
22  
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  import java.util.ArrayList;
26  import java.util.List;
27  import java.util.Map;
28  
29  import org.apache.hadoop.classification.InterfaceAudience;
30  import org.apache.hadoop.hbase.Cell;
31  import org.apache.hadoop.hbase.CellUtil;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.KeyValue;
34  import org.apache.hadoop.hbase.KeyValueUtil;
35  import org.apache.hadoop.hbase.client.Append;
36  import org.apache.hadoop.hbase.client.Delete;
37  import org.apache.hadoop.hbase.client.Durability;
38  import org.apache.hadoop.hbase.client.Get;
39  import org.apache.hadoop.hbase.client.Increment;
40  import org.apache.hadoop.hbase.client.OperationWithAttributes;
41  import org.apache.hadoop.hbase.client.Put;
42  import org.apache.hadoop.hbase.client.Result;
43  import org.apache.hadoop.hbase.client.RowMutations;
44  import org.apache.hadoop.hbase.client.Scan;
45  import org.apache.hadoop.hbase.filter.ParseFilter;
46  import org.apache.hadoop.hbase.security.visibility.Authorizations;
47  import org.apache.hadoop.hbase.security.visibility.CellVisibility;
48  import org.apache.hadoop.hbase.thrift2.generated.TAppend;
49  import org.apache.hadoop.hbase.thrift2.generated.TColumn;
50  import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
51  import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
52  import org.apache.hadoop.hbase.thrift2.generated.TDelete;
53  import org.apache.hadoop.hbase.thrift2.generated.TDeleteType;
54  import org.apache.hadoop.hbase.thrift2.generated.TDurability;
55  import org.apache.hadoop.hbase.thrift2.generated.TGet;
56  import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
57  import org.apache.hadoop.hbase.thrift2.generated.TMutation;
58  import org.apache.hadoop.hbase.thrift2.generated.TPut;
59  import org.apache.hadoop.hbase.thrift2.generated.TResult;
60  import org.apache.hadoop.hbase.thrift2.generated.TRowMutations;
61  import org.apache.hadoop.hbase.thrift2.generated.TScan;
62  import org.apache.hadoop.hbase.thrift2.generated.TTimeRange;
63  import org.apache.hadoop.hbase.util.Bytes;
64  
65  @InterfaceAudience.Private
66  public class ThriftUtilities {
67  
68    private ThriftUtilities() {
69      throw new UnsupportedOperationException("Can't initialize class");
70    }
71  
72    /**
73     * Creates a {@link Get} (HBase) from a {@link TGet} (Thrift).
74     *
75     * This ignores any timestamps set on {@link TColumn} objects.
76     *
77     * @param in the <code>TGet</code> to convert
78     *
79     * @return <code>Get</code> object
80     *
81     * @throws IOException if an invalid time range or max version parameter is given
82     */
83    public static Get getFromThrift(TGet in) throws IOException {
84      Get out = new Get(in.getRow());
85  
86      // Timestamp overwrites time range if both are set
87      if (in.isSetTimestamp()) {
88        out.setTimeStamp(in.getTimestamp());
89      } else if (in.isSetTimeRange()) {
90        out.setTimeRange(in.getTimeRange().getMinStamp(), in.getTimeRange().getMaxStamp());
91      }
92  
93      if (in.isSetMaxVersions()) {
94        out.setMaxVersions(in.getMaxVersions());
95      }
96  
97      if (in.isSetFilterString()) {
98        ParseFilter parseFilter = new ParseFilter();
99        out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
100     }
101 
102     if (in.isSetAttributes()) {
103       addAttributes(out,in.getAttributes());
104     }
105 
106     if (in.isSetAuthorizations()) {
107       out.setAuthorizations(new Authorizations(in.getAuthorizations().getLabels()));
108     }
109     
110     if (!in.isSetColumns()) {
111       return out;
112     }
113 
114     for (TColumn column : in.getColumns()) {
115       if (column.isSetQualifier()) {
116         out.addColumn(column.getFamily(), column.getQualifier());
117       } else {
118         out.addFamily(column.getFamily());
119       }
120     }
121 
122     return out;
123   }
124 
125   /**
126    * Converts multiple {@link TGet}s (Thrift) into a list of {@link Get}s (HBase).
127    *
128    * @param in list of <code>TGet</code>s to convert
129    *
130    * @return list of <code>Get</code> objects
131    *
132    * @throws IOException if an invalid time range or max version parameter is given
133    * @see #getFromThrift(TGet)
134    */
135   public static List<Get> getsFromThrift(List<TGet> in) throws IOException {
136     List<Get> out = new ArrayList<Get>(in.size());
137     for (TGet get : in) {
138       out.add(getFromThrift(get));
139     }
140     return out;
141   }
142 
143   /**
144    * Creates a {@link TResult} (Thrift) from a {@link Result} (HBase).
145    *
146    * @param in the <code>Result</code> to convert
147    *
148    * @return converted result, returns an empty result if the input is <code>null</code>
149    */
150   public static TResult resultFromHBase(Result in) {
151     Cell[] raw = in.rawCells();
152     TResult out = new TResult();
153     byte[] row = in.getRow();
154     if (row != null) {
155       out.setRow(in.getRow());
156     }
157     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
158     for (Cell kv : raw) {
159       TColumnValue col = new TColumnValue();
160       col.setFamily(CellUtil.cloneFamily(kv));
161       col.setQualifier(CellUtil.cloneQualifier(kv));
162       col.setTimestamp(kv.getTimestamp());
163       col.setValue(CellUtil.cloneValue(kv));
164       if (kv.getTagsLength() > 0) {
165         col.setTags(CellUtil.getTagArray(kv));
166       }
167       columnValues.add(col);
168     }
169     out.setColumnValues(columnValues);
170     return out;
171   }
172 
173   /**
174    * Converts multiple {@link Result}s (HBase) into a list of {@link TResult}s (Thrift).
175    *
176    * @param in array of <code>Result</code>s to convert
177    *
178    * @return list of converted <code>TResult</code>s
179    *
180    * @see #resultFromHBase(Result)
181    */
182   public static List<TResult> resultsFromHBase(Result[] in) {
183     List<TResult> out = new ArrayList<TResult>(in.length);
184     for (Result result : in) {
185       out.add(resultFromHBase(result));
186     }
187     return out;
188   }
189 
190   /**
191    * Creates a {@link Put} (HBase) from a {@link TPut} (Thrift)
192    *
193    * @param in the <code>TPut</code> to convert
194    *
195    * @return converted <code>Put</code>
196    */
197   public static Put putFromThrift(TPut in) {
198     Put out;
199 
200     if (in.isSetTimestamp()) {
201       out = new Put(in.getRow(), in.getTimestamp());
202     } else {
203       out = new Put(in.getRow());
204     }
205 
206     if (in.isSetDurability()) {
207       out.setDurability(durabilityFromThrift(in.getDurability()));
208     }
209 
210     for (TColumnValue columnValue : in.getColumnValues()) {
211       if (columnValue.isSetTimestamp()) {
212         out.addImmutable(
213             columnValue.getFamily(), columnValue.getQualifier(), columnValue.getTimestamp(),
214             columnValue.getValue());
215       } else {
216         out.addImmutable(
217             columnValue.getFamily(), columnValue.getQualifier(), columnValue.getValue());
218       }
219     }
220 
221     if (in.isSetAttributes()) {
222       addAttributes(out,in.getAttributes());
223     }
224     
225     if (in.getCellVisibility() != null) {
226       out.setCellVisibility(new CellVisibility(in.getCellVisibility().getExpression()));
227     }
228 
229     return out;
230   }
231 
232   /**
233    * Converts multiple {@link TPut}s (Thrift) into a list of {@link Put}s (HBase).
234    *
235    * @param in list of <code>TPut</code>s to convert
236    *
237    * @return list of converted <code>Put</code>s
238    *
239    * @see #putFromThrift(TPut)
240    */
241   public static List<Put> putsFromThrift(List<TPut> in) {
242     List<Put> out = new ArrayList<Put>(in.size());
243     for (TPut put : in) {
244       out.add(putFromThrift(put));
245     }
246     return out;
247   }
248 
249   /**
250    * Creates a {@link Delete} (HBase) from a {@link TDelete} (Thrift).
251    *
252    * @param in the <code>TDelete</code> to convert
253    *
254    * @return converted <code>Delete</code>
255    */
256   public static Delete deleteFromThrift(TDelete in) {
257     Delete out;
258 
259     if (in.isSetColumns()) {
260       out = new Delete(in.getRow());
261       for (TColumn column : in.getColumns()) {
262         if (column.isSetQualifier()) {
263           if (column.isSetTimestamp()) {
264             if (in.isSetDeleteType() &&
265                 in.getDeleteType().equals(TDeleteType.DELETE_COLUMNS))
266               out.deleteColumns(column.getFamily(), column.getQualifier(), column.getTimestamp());
267             else
268               out.deleteColumn(column.getFamily(), column.getQualifier(), column.getTimestamp());
269           } else {
270             if (in.isSetDeleteType() &&
271                 in.getDeleteType().equals(TDeleteType.DELETE_COLUMNS))
272               out.deleteColumns(column.getFamily(), column.getQualifier());
273             else
274               out.deleteColumn(column.getFamily(), column.getQualifier());
275           }
276 
277         } else {
278           if (column.isSetTimestamp()) {
279             out.deleteFamily(column.getFamily(), column.getTimestamp());
280           } else {
281             out.deleteFamily(column.getFamily());
282           }
283         }
284       }
285     } else {
286       if (in.isSetTimestamp()) {
287         out = new Delete(in.getRow(), in.getTimestamp());
288       } else {
289         out = new Delete(in.getRow());
290       }
291     }
292 
293     if (in.isSetAttributes()) {
294       addAttributes(out,in.getAttributes());
295     }
296 
297     if (in.isSetDurability()) {
298       out.setDurability(durabilityFromThrift(in.getDurability()));
299     }
300 
301     return out;
302   }
303 
304   /**
305    * Converts multiple {@link TDelete}s (Thrift) into a list of {@link Delete}s (HBase).
306    *
307    * @param in list of <code>TDelete</code>s to convert
308    *
309    * @return list of converted <code>Delete</code>s
310    *
311    * @see #deleteFromThrift(TDelete)
312    */
313 
314   public static List<Delete> deletesFromThrift(List<TDelete> in) {
315     List<Delete> out = new ArrayList<Delete>(in.size());
316     for (TDelete delete : in) {
317       out.add(deleteFromThrift(delete));
318     }
319     return out;
320   }
321 
322   public static TDelete deleteFromHBase(Delete in) {
323     TDelete out = new TDelete(ByteBuffer.wrap(in.getRow()));
324 
325     List<TColumn> columns = new ArrayList<TColumn>();
326     long rowTimestamp = in.getTimeStamp();
327     if (rowTimestamp != HConstants.LATEST_TIMESTAMP) {
328       out.setTimestamp(rowTimestamp);
329     }
330 
331     // Map<family, List<KeyValue>>
332     for (Map.Entry<byte[], List<org.apache.hadoop.hbase.Cell>> familyEntry:
333         in.getFamilyCellMap().entrySet()) {
334       TColumn column = new TColumn(ByteBuffer.wrap(familyEntry.getKey()));
335       for (org.apache.hadoop.hbase.Cell cell: familyEntry.getValue()) {
336         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
337         byte[] family = kv.getFamily();
338         byte[] qualifier = kv.getQualifier();
339         long timestamp = kv.getTimestamp();
340         if (family != null) {
341           column.setFamily(family);
342         }
343         if (qualifier != null) {
344           column.setQualifier(qualifier);
345         }
346         if (timestamp != HConstants.LATEST_TIMESTAMP) {
347           column.setTimestamp(kv.getTimestamp());
348         }
349       }
350       columns.add(column);
351     }
352     out.setColumns(columns);
353 
354     return out;
355   }
356 
357   /**
358    * Creates a {@link RowMutations} (HBase) from a {@link TRowMutations} (Thrift)
359    *
360    * @param in the <code>TRowMutations</code> to convert
361    *
362    * @return converted <code>RowMutations</code>
363    */
364   public static RowMutations rowMutationsFromThrift(TRowMutations in) throws IOException {
365     RowMutations out = new RowMutations(in.getRow());
366     List<TMutation> mutations = in.getMutations();
367     for (TMutation mutation : mutations) {
368       if (mutation.isSetPut()) {
369         out.add(putFromThrift(mutation.getPut()));
370       }
371       if (mutation.isSetDeleteSingle()) {
372         out.add(deleteFromThrift(mutation.getDeleteSingle()));
373       }
374     }
375     return out;
376   }
377 
378   public static Scan scanFromThrift(TScan in) throws IOException {
379     Scan out = new Scan();
380 
381     if (in.isSetStartRow())
382       out.setStartRow(in.getStartRow());
383     if (in.isSetStopRow())
384       out.setStopRow(in.getStopRow());
385     if (in.isSetCaching())
386       out.setCaching(in.getCaching());
387     if (in.isSetMaxVersions()) {
388       out.setMaxVersions(in.getMaxVersions());
389     }
390 
391     if (in.isSetColumns()) {
392       for (TColumn column : in.getColumns()) {
393         if (column.isSetQualifier()) {
394           out.addColumn(column.getFamily(), column.getQualifier());
395         } else {
396           out.addFamily(column.getFamily());
397         }
398       }
399     }
400 
401     TTimeRange timeRange = in.getTimeRange();
402     if (timeRange != null &&
403         timeRange.isSetMinStamp() && timeRange.isSetMaxStamp()) {
404       out.setTimeRange(timeRange.getMinStamp(), timeRange.getMaxStamp());
405     }
406 
407     if (in.isSetBatchSize()) {
408       out.setBatch(in.getBatchSize());
409     }
410 
411     if (in.isSetFilterString()) {
412       ParseFilter parseFilter = new ParseFilter();
413       out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
414     }
415 
416     if (in.isSetAttributes()) {
417       addAttributes(out,in.getAttributes());
418     }
419     
420 		if (in.isSetAuthorizations()) {
421 		  out.setAuthorizations(new Authorizations(in.getAuthorizations().getLabels()));
422     }
423 
424     return out;
425   }
426 
427   public static Increment incrementFromThrift(TIncrement in) throws IOException {
428     Increment out = new Increment(in.getRow());
429     for (TColumnIncrement column : in.getColumns()) {
430       out.addColumn(column.getFamily(), column.getQualifier(), column.getAmount());
431     }
432 
433     if (in.isSetAttributes()) {
434       addAttributes(out,in.getAttributes());
435     }
436 
437     if (in.isSetDurability()) {
438       out.setDurability(durabilityFromThrift(in.getDurability()));
439     }
440     
441     if(in.getCellVisibility() != null) {
442       out.setCellVisibility(new CellVisibility(in.getCellVisibility().getExpression()));
443     }
444 
445     return out;
446   }
447 
448   public static Append appendFromThrift(TAppend append) throws IOException {
449     Append out = new Append(append.getRow());
450     for (TColumnValue column : append.getColumns()) {
451       out.add(column.getFamily(), column.getQualifier(), column.getValue());
452     }
453 
454     if (append.isSetAttributes()) {
455       addAttributes(out, append.getAttributes());
456     }
457 
458     if (append.isSetDurability()) {
459       out.setDurability(durabilityFromThrift(append.getDurability()));
460     }
461     
462     if(append.getCellVisibility() != null) {
463       out.setCellVisibility(new CellVisibility(append.getCellVisibility().getExpression()));
464     }
465 
466     return out;
467   }
468 
469   /**
470    * Adds all the attributes into the Operation object
471    */
472   private static void addAttributes(OperationWithAttributes op,
473                                     Map<ByteBuffer, ByteBuffer> attributes) {
474     if (attributes == null || attributes.size() == 0) {
475       return;
476     }
477     for (Map.Entry<ByteBuffer, ByteBuffer> entry : attributes.entrySet()) {
478       String name = Bytes.toStringBinary(getBytes(entry.getKey()));
479       byte[] value =  getBytes(entry.getValue());
480       op.setAttribute(name, value);
481     }
482   }
483 
484   private static Durability durabilityFromThrift(TDurability tDurability) {
485     switch (tDurability.getValue()) {
486       case 1: return Durability.SKIP_WAL;
487       case 2: return Durability.ASYNC_WAL;
488       case 3: return Durability.SYNC_WAL;
489       case 4: return Durability.FSYNC_WAL;
490       default: return null;
491     }
492   }
493 }