View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.thrift2;
20  
21  import static org.apache.hadoop.hbase.util.Bytes.getBytes;
22  
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  import java.util.ArrayList;
26  import java.util.List;
27  import java.util.Map;
28  
29  import org.apache.hadoop.hbase.classification.InterfaceAudience;
30  import org.apache.hadoop.hbase.Cell;
31  import org.apache.hadoop.hbase.CellUtil;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.client.Append;
34  import org.apache.hadoop.hbase.client.Delete;
35  import org.apache.hadoop.hbase.client.Durability;
36  import org.apache.hadoop.hbase.client.Get;
37  import org.apache.hadoop.hbase.client.Increment;
38  import org.apache.hadoop.hbase.client.OperationWithAttributes;
39  import org.apache.hadoop.hbase.client.Put;
40  import org.apache.hadoop.hbase.client.Result;
41  import org.apache.hadoop.hbase.client.RowMutations;
42  import org.apache.hadoop.hbase.client.Scan;
43  import org.apache.hadoop.hbase.filter.ParseFilter;
44  import org.apache.hadoop.hbase.security.visibility.Authorizations;
45  import org.apache.hadoop.hbase.security.visibility.CellVisibility;
46  import org.apache.hadoop.hbase.thrift2.generated.TAppend;
47  import org.apache.hadoop.hbase.thrift2.generated.TColumn;
48  import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
49  import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
50  import org.apache.hadoop.hbase.thrift2.generated.TDelete;
51  import org.apache.hadoop.hbase.thrift2.generated.TDeleteType;
52  import org.apache.hadoop.hbase.thrift2.generated.TDurability;
53  import org.apache.hadoop.hbase.thrift2.generated.TGet;
54  import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
55  import org.apache.hadoop.hbase.thrift2.generated.TMutation;
56  import org.apache.hadoop.hbase.thrift2.generated.TPut;
57  import org.apache.hadoop.hbase.thrift2.generated.TResult;
58  import org.apache.hadoop.hbase.thrift2.generated.TRowMutations;
59  import org.apache.hadoop.hbase.thrift2.generated.TScan;
60  import org.apache.hadoop.hbase.thrift2.generated.TTimeRange;
61  import org.apache.hadoop.hbase.util.Bytes;
62  
63  @InterfaceAudience.Private
64  public class ThriftUtilities {
65  
66    private ThriftUtilities() {
67      throw new UnsupportedOperationException("Can't initialize class");
68    }
69  
70    /**
71     * Creates a {@link Get} (HBase) from a {@link TGet} (Thrift).
72     *
73     * This ignores any timestamps set on {@link TColumn} objects.
74     *
75     * @param in the <code>TGet</code> to convert
76     *
77     * @return <code>Get</code> object
78     *
79     * @throws IOException if an invalid time range or max version parameter is given
80     */
81    public static Get getFromThrift(TGet in) throws IOException {
82      Get out = new Get(in.getRow());
83  
84      // Timestamp overwrites time range if both are set
85      if (in.isSetTimestamp()) {
86        out.setTimeStamp(in.getTimestamp());
87      } else if (in.isSetTimeRange()) {
88        out.setTimeRange(in.getTimeRange().getMinStamp(), in.getTimeRange().getMaxStamp());
89      }
90  
91      if (in.isSetMaxVersions()) {
92        out.setMaxVersions(in.getMaxVersions());
93      }
94  
95      if (in.isSetFilterString()) {
96        ParseFilter parseFilter = new ParseFilter();
97        out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
98      }
99  
100     if (in.isSetAttributes()) {
101       addAttributes(out,in.getAttributes());
102     }
103 
104     if (in.isSetAuthorizations()) {
105       out.setAuthorizations(new Authorizations(in.getAuthorizations().getLabels()));
106     }
107     
108     if (!in.isSetColumns()) {
109       return out;
110     }
111 
112     for (TColumn column : in.getColumns()) {
113       if (column.isSetQualifier()) {
114         out.addColumn(column.getFamily(), column.getQualifier());
115       } else {
116         out.addFamily(column.getFamily());
117       }
118     }
119 
120     return out;
121   }
122 
123   /**
124    * Converts multiple {@link TGet}s (Thrift) into a list of {@link Get}s (HBase).
125    *
126    * @param in list of <code>TGet</code>s to convert
127    *
128    * @return list of <code>Get</code> objects
129    *
130    * @throws IOException if an invalid time range or max version parameter is given
131    * @see #getFromThrift(TGet)
132    */
133   public static List<Get> getsFromThrift(List<TGet> in) throws IOException {
134     List<Get> out = new ArrayList<Get>(in.size());
135     for (TGet get : in) {
136       out.add(getFromThrift(get));
137     }
138     return out;
139   }
140 
141   /**
142    * Creates a {@link TResult} (Thrift) from a {@link Result} (HBase).
143    *
144    * @param in the <code>Result</code> to convert
145    *
146    * @return converted result, returns an empty result if the input is <code>null</code>
147    */
148   public static TResult resultFromHBase(Result in) {
149     Cell[] raw = in.rawCells();
150     TResult out = new TResult();
151     byte[] row = in.getRow();
152     if (row != null) {
153       out.setRow(in.getRow());
154     }
155     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
156     for (Cell kv : raw) {
157       TColumnValue col = new TColumnValue();
158       col.setFamily(CellUtil.cloneFamily(kv));
159       col.setQualifier(CellUtil.cloneQualifier(kv));
160       col.setTimestamp(kv.getTimestamp());
161       col.setValue(CellUtil.cloneValue(kv));
162       if (kv.getTagsLength() > 0) {
163         col.setTags(CellUtil.getTagArray(kv));
164       }
165       columnValues.add(col);
166     }
167     out.setColumnValues(columnValues);
168     return out;
169   }
170 
171   /**
172    * Converts multiple {@link Result}s (HBase) into a list of {@link TResult}s (Thrift).
173    *
174    * @param in array of <code>Result</code>s to convert
175    *
176    * @return list of converted <code>TResult</code>s
177    *
178    * @see #resultFromHBase(Result)
179    */
180   public static List<TResult> resultsFromHBase(Result[] in) {
181     List<TResult> out = new ArrayList<TResult>(in.length);
182     for (Result result : in) {
183       out.add(resultFromHBase(result));
184     }
185     return out;
186   }
187 
188   /**
189    * Creates a {@link Put} (HBase) from a {@link TPut} (Thrift)
190    *
191    * @param in the <code>TPut</code> to convert
192    *
193    * @return converted <code>Put</code>
194    */
195   public static Put putFromThrift(TPut in) {
196     Put out;
197 
198     if (in.isSetTimestamp()) {
199       out = new Put(in.getRow(), in.getTimestamp());
200     } else {
201       out = new Put(in.getRow());
202     }
203 
204     if (in.isSetDurability()) {
205       out.setDurability(durabilityFromThrift(in.getDurability()));
206     }
207 
208     for (TColumnValue columnValue : in.getColumnValues()) {
209       if (columnValue.isSetTimestamp()) {
210         out.addImmutable(
211             columnValue.getFamily(), columnValue.getQualifier(), columnValue.getTimestamp(),
212             columnValue.getValue());
213       } else {
214         out.addImmutable(
215             columnValue.getFamily(), columnValue.getQualifier(), columnValue.getValue());
216       }
217     }
218 
219     if (in.isSetAttributes()) {
220       addAttributes(out,in.getAttributes());
221     }
222     
223     if (in.getCellVisibility() != null) {
224       out.setCellVisibility(new CellVisibility(in.getCellVisibility().getExpression()));
225     }
226 
227     return out;
228   }
229 
230   /**
231    * Converts multiple {@link TPut}s (Thrift) into a list of {@link Put}s (HBase).
232    *
233    * @param in list of <code>TPut</code>s to convert
234    *
235    * @return list of converted <code>Put</code>s
236    *
237    * @see #putFromThrift(TPut)
238    */
239   public static List<Put> putsFromThrift(List<TPut> in) {
240     List<Put> out = new ArrayList<Put>(in.size());
241     for (TPut put : in) {
242       out.add(putFromThrift(put));
243     }
244     return out;
245   }
246 
247   /**
248    * Creates a {@link Delete} (HBase) from a {@link TDelete} (Thrift).
249    *
250    * @param in the <code>TDelete</code> to convert
251    *
252    * @return converted <code>Delete</code>
253    */
254   public static Delete deleteFromThrift(TDelete in) {
255     Delete out;
256 
257     if (in.isSetColumns()) {
258       out = new Delete(in.getRow());
259       for (TColumn column : in.getColumns()) {
260         if (column.isSetQualifier()) {
261           if (column.isSetTimestamp()) {
262             if (in.isSetDeleteType() &&
263                 in.getDeleteType().equals(TDeleteType.DELETE_COLUMNS))
264               out.deleteColumns(column.getFamily(), column.getQualifier(), column.getTimestamp());
265             else
266               out.deleteColumn(column.getFamily(), column.getQualifier(), column.getTimestamp());
267           } else {
268             if (in.isSetDeleteType() &&
269                 in.getDeleteType().equals(TDeleteType.DELETE_COLUMNS))
270               out.deleteColumns(column.getFamily(), column.getQualifier());
271             else
272               out.deleteColumn(column.getFamily(), column.getQualifier());
273           }
274 
275         } else {
276           if (column.isSetTimestamp()) {
277             out.deleteFamily(column.getFamily(), column.getTimestamp());
278           } else {
279             out.deleteFamily(column.getFamily());
280           }
281         }
282       }
283     } else {
284       if (in.isSetTimestamp()) {
285         out = new Delete(in.getRow(), in.getTimestamp());
286       } else {
287         out = new Delete(in.getRow());
288       }
289     }
290 
291     if (in.isSetAttributes()) {
292       addAttributes(out,in.getAttributes());
293     }
294 
295     if (in.isSetDurability()) {
296       out.setDurability(durabilityFromThrift(in.getDurability()));
297     }
298 
299     return out;
300   }
301 
302   /**
303    * Converts multiple {@link TDelete}s (Thrift) into a list of {@link Delete}s (HBase).
304    *
305    * @param in list of <code>TDelete</code>s to convert
306    *
307    * @return list of converted <code>Delete</code>s
308    *
309    * @see #deleteFromThrift(TDelete)
310    */
311 
312   public static List<Delete> deletesFromThrift(List<TDelete> in) {
313     List<Delete> out = new ArrayList<Delete>(in.size());
314     for (TDelete delete : in) {
315       out.add(deleteFromThrift(delete));
316     }
317     return out;
318   }
319 
320   public static TDelete deleteFromHBase(Delete in) {
321     TDelete out = new TDelete(ByteBuffer.wrap(in.getRow()));
322 
323     List<TColumn> columns = new ArrayList<TColumn>();
324     long rowTimestamp = in.getTimeStamp();
325     if (rowTimestamp != HConstants.LATEST_TIMESTAMP) {
326       out.setTimestamp(rowTimestamp);
327     }
328 
329     // Map<family, List<KeyValue>>
330     for (Map.Entry<byte[], List<org.apache.hadoop.hbase.Cell>> familyEntry:
331         in.getFamilyCellMap().entrySet()) {
332       TColumn column = new TColumn(ByteBuffer.wrap(familyEntry.getKey()));
333       for (org.apache.hadoop.hbase.Cell cell: familyEntry.getValue()) {
334         byte[] family = CellUtil.cloneFamily(cell);
335         byte[] qualifier = CellUtil.cloneQualifier(cell);
336         long timestamp = cell.getTimestamp();
337         if (family != null) {
338           column.setFamily(family);
339         }
340         if (qualifier != null) {
341           column.setQualifier(qualifier);
342         }
343         if (timestamp != HConstants.LATEST_TIMESTAMP) {
344           column.setTimestamp(timestamp);
345         }
346       }
347       columns.add(column);
348     }
349     out.setColumns(columns);
350 
351     return out;
352   }
353 
354   /**
355    * Creates a {@link RowMutations} (HBase) from a {@link TRowMutations} (Thrift)
356    *
357    * @param in the <code>TRowMutations</code> to convert
358    *
359    * @return converted <code>RowMutations</code>
360    */
361   public static RowMutations rowMutationsFromThrift(TRowMutations in) throws IOException {
362     RowMutations out = new RowMutations(in.getRow());
363     List<TMutation> mutations = in.getMutations();
364     for (TMutation mutation : mutations) {
365       if (mutation.isSetPut()) {
366         out.add(putFromThrift(mutation.getPut()));
367       }
368       if (mutation.isSetDeleteSingle()) {
369         out.add(deleteFromThrift(mutation.getDeleteSingle()));
370       }
371     }
372     return out;
373   }
374 
375   public static Scan scanFromThrift(TScan in) throws IOException {
376     Scan out = new Scan();
377 
378     if (in.isSetStartRow())
379       out.setStartRow(in.getStartRow());
380     if (in.isSetStopRow())
381       out.setStopRow(in.getStopRow());
382     if (in.isSetCaching())
383       out.setCaching(in.getCaching());
384     if (in.isSetMaxVersions()) {
385       out.setMaxVersions(in.getMaxVersions());
386     }
387 
388     if (in.isSetColumns()) {
389       for (TColumn column : in.getColumns()) {
390         if (column.isSetQualifier()) {
391           out.addColumn(column.getFamily(), column.getQualifier());
392         } else {
393           out.addFamily(column.getFamily());
394         }
395       }
396     }
397 
398     TTimeRange timeRange = in.getTimeRange();
399     if (timeRange != null &&
400         timeRange.isSetMinStamp() && timeRange.isSetMaxStamp()) {
401       out.setTimeRange(timeRange.getMinStamp(), timeRange.getMaxStamp());
402     }
403 
404     if (in.isSetBatchSize()) {
405       out.setBatch(in.getBatchSize());
406     }
407 
408     if (in.isSetFilterString()) {
409       ParseFilter parseFilter = new ParseFilter();
410       out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
411     }
412 
413     if (in.isSetAttributes()) {
414       addAttributes(out,in.getAttributes());
415     }
416     
417     if (in.isSetAuthorizations()) {
418       out.setAuthorizations(new Authorizations(in.getAuthorizations().getLabels()));
419     }
420 
421     if (in.isSetReversed()) {
422       out.setReversed(in.isReversed());
423     }
424 
425     return out;
426   }
427 
428   public static Increment incrementFromThrift(TIncrement in) throws IOException {
429     Increment out = new Increment(in.getRow());
430     for (TColumnIncrement column : in.getColumns()) {
431       out.addColumn(column.getFamily(), column.getQualifier(), column.getAmount());
432     }
433 
434     if (in.isSetAttributes()) {
435       addAttributes(out,in.getAttributes());
436     }
437 
438     if (in.isSetDurability()) {
439       out.setDurability(durabilityFromThrift(in.getDurability()));
440     }
441     
442     if(in.getCellVisibility() != null) {
443       out.setCellVisibility(new CellVisibility(in.getCellVisibility().getExpression()));
444     }
445 
446     return out;
447   }
448 
449   public static Append appendFromThrift(TAppend append) throws IOException {
450     Append out = new Append(append.getRow());
451     for (TColumnValue column : append.getColumns()) {
452       out.add(column.getFamily(), column.getQualifier(), column.getValue());
453     }
454 
455     if (append.isSetAttributes()) {
456       addAttributes(out, append.getAttributes());
457     }
458 
459     if (append.isSetDurability()) {
460       out.setDurability(durabilityFromThrift(append.getDurability()));
461     }
462     
463     if(append.getCellVisibility() != null) {
464       out.setCellVisibility(new CellVisibility(append.getCellVisibility().getExpression()));
465     }
466 
467     return out;
468   }
469 
470   /**
471    * Adds all the attributes into the Operation object
472    */
473   private static void addAttributes(OperationWithAttributes op,
474                                     Map<ByteBuffer, ByteBuffer> attributes) {
475     if (attributes == null || attributes.size() == 0) {
476       return;
477     }
478     for (Map.Entry<ByteBuffer, ByteBuffer> entry : attributes.entrySet()) {
479       String name = Bytes.toStringBinary(getBytes(entry.getKey()));
480       byte[] value =  getBytes(entry.getValue());
481       op.setAttribute(name, value);
482     }
483   }
484 
485   private static Durability durabilityFromThrift(TDurability tDurability) {
486     switch (tDurability.getValue()) {
487       case 1: return Durability.SKIP_WAL;
488       case 2: return Durability.ASYNC_WAL;
489       case 3: return Durability.SYNC_WAL;
490       case 4: return Durability.FSYNC_WAL;
491       default: return null;
492     }
493   }
494 }