View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.thrift2;
20  
21  import static org.apache.hadoop.hbase.util.Bytes.getBytes;
22  
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  import java.util.ArrayList;
26  import java.util.List;
27  import java.util.Map;
28  
29  import org.apache.hadoop.hbase.HRegionInfo;
30  import org.apache.hadoop.hbase.HRegionLocation;
31  import org.apache.hadoop.hbase.ServerName;
32  import org.apache.hadoop.hbase.classification.InterfaceAudience;
33  import org.apache.hadoop.hbase.Cell;
34  import org.apache.hadoop.hbase.CellUtil;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.client.Append;
37  import org.apache.hadoop.hbase.client.Delete;
38  import org.apache.hadoop.hbase.client.Durability;
39  import org.apache.hadoop.hbase.client.Get;
40  import org.apache.hadoop.hbase.client.Increment;
41  import org.apache.hadoop.hbase.client.OperationWithAttributes;
42  import org.apache.hadoop.hbase.client.Put;
43  import org.apache.hadoop.hbase.client.Result;
44  import org.apache.hadoop.hbase.client.RowMutations;
45  import org.apache.hadoop.hbase.client.Scan;
46  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
47  import org.apache.hadoop.hbase.filter.ParseFilter;
48  import org.apache.hadoop.hbase.security.visibility.Authorizations;
49  import org.apache.hadoop.hbase.security.visibility.CellVisibility;
50  import org.apache.hadoop.hbase.thrift2.generated.TAppend;
51  import org.apache.hadoop.hbase.thrift2.generated.TColumn;
52  import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
53  import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
54  import org.apache.hadoop.hbase.thrift2.generated.TCompareOp;
55  import org.apache.hadoop.hbase.thrift2.generated.TDelete;
56  import org.apache.hadoop.hbase.thrift2.generated.TDeleteType;
57  import org.apache.hadoop.hbase.thrift2.generated.TDurability;
58  import org.apache.hadoop.hbase.thrift2.generated.TGet;
59  import org.apache.hadoop.hbase.thrift2.generated.THRegionInfo;
60  import org.apache.hadoop.hbase.thrift2.generated.THRegionLocation;
61  import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
62  import org.apache.hadoop.hbase.thrift2.generated.TMutation;
63  import org.apache.hadoop.hbase.thrift2.generated.TPut;
64  import org.apache.hadoop.hbase.thrift2.generated.TResult;
65  import org.apache.hadoop.hbase.thrift2.generated.TRowMutations;
66  import org.apache.hadoop.hbase.thrift2.generated.TScan;
67  import org.apache.hadoop.hbase.thrift2.generated.TServerName;
68  import org.apache.hadoop.hbase.thrift2.generated.TTimeRange;
69  import org.apache.hadoop.hbase.util.Bytes;
70  
71  @InterfaceAudience.Private
72  public class ThriftUtilities {
73  
74    private ThriftUtilities() {
75      throw new UnsupportedOperationException("Can't initialize class");
76    }
77  
78    /**
79     * Creates a {@link Get} (HBase) from a {@link TGet} (Thrift).
80     *
81     * This ignores any timestamps set on {@link TColumn} objects.
82     *
83     * @param in the <code>TGet</code> to convert
84     *
85     * @return <code>Get</code> object
86     *
87     * @throws IOException if an invalid time range or max version parameter is given
88     */
89    public static Get getFromThrift(TGet in) throws IOException {
90      Get out = new Get(in.getRow());
91  
92      // Timestamp overwrites time range if both are set
93      if (in.isSetTimestamp()) {
94        out.setTimeStamp(in.getTimestamp());
95      } else if (in.isSetTimeRange()) {
96        out.setTimeRange(in.getTimeRange().getMinStamp(), in.getTimeRange().getMaxStamp());
97      }
98  
99      if (in.isSetMaxVersions()) {
100       out.setMaxVersions(in.getMaxVersions());
101     }
102 
103     if (in.isSetFilterString()) {
104       ParseFilter parseFilter = new ParseFilter();
105       out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
106     }
107 
108     if (in.isSetAttributes()) {
109       addAttributes(out,in.getAttributes());
110     }
111 
112     if (in.isSetAuthorizations()) {
113       out.setAuthorizations(new Authorizations(in.getAuthorizations().getLabels()));
114     }
115     
116     if (!in.isSetColumns()) {
117       return out;
118     }
119 
120     for (TColumn column : in.getColumns()) {
121       if (column.isSetQualifier()) {
122         out.addColumn(column.getFamily(), column.getQualifier());
123       } else {
124         out.addFamily(column.getFamily());
125       }
126     }
127 
128     return out;
129   }
130 
131   /**
132    * Converts multiple {@link TGet}s (Thrift) into a list of {@link Get}s (HBase).
133    *
134    * @param in list of <code>TGet</code>s to convert
135    *
136    * @return list of <code>Get</code> objects
137    *
138    * @throws IOException if an invalid time range or max version parameter is given
139    * @see #getFromThrift(TGet)
140    */
141   public static List<Get> getsFromThrift(List<TGet> in) throws IOException {
142     List<Get> out = new ArrayList<Get>(in.size());
143     for (TGet get : in) {
144       out.add(getFromThrift(get));
145     }
146     return out;
147   }
148 
149   /**
150    * Creates a {@link TResult} (Thrift) from a {@link Result} (HBase).
151    *
152    * @param in the <code>Result</code> to convert
153    *
154    * @return converted result, returns an empty result if the input is <code>null</code>
155    */
156   public static TResult resultFromHBase(Result in) {
157     Cell[] raw = in.rawCells();
158     TResult out = new TResult();
159     byte[] row = in.getRow();
160     if (row != null) {
161       out.setRow(in.getRow());
162     }
163     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
164     for (Cell kv : raw) {
165       TColumnValue col = new TColumnValue();
166       col.setFamily(CellUtil.cloneFamily(kv));
167       col.setQualifier(CellUtil.cloneQualifier(kv));
168       col.setTimestamp(kv.getTimestamp());
169       col.setValue(CellUtil.cloneValue(kv));
170       if (kv.getTagsLength() > 0) {
171         col.setTags(CellUtil.getTagArray(kv));
172       }
173       columnValues.add(col);
174     }
175     out.setColumnValues(columnValues);
176     return out;
177   }
178 
179   /**
180    * Converts multiple {@link Result}s (HBase) into a list of {@link TResult}s (Thrift).
181    *
182    * @param in array of <code>Result</code>s to convert
183    *
184    * @return list of converted <code>TResult</code>s
185    *
186    * @see #resultFromHBase(Result)
187    */
188   public static List<TResult> resultsFromHBase(Result[] in) {
189     List<TResult> out = new ArrayList<TResult>(in.length);
190     for (Result result : in) {
191       out.add(resultFromHBase(result));
192     }
193     return out;
194   }
195 
196   /**
197    * Creates a {@link Put} (HBase) from a {@link TPut} (Thrift)
198    *
199    * @param in the <code>TPut</code> to convert
200    *
201    * @return converted <code>Put</code>
202    */
203   public static Put putFromThrift(TPut in) {
204     Put out;
205 
206     if (in.isSetTimestamp()) {
207       out = new Put(in.getRow(), in.getTimestamp());
208     } else {
209       out = new Put(in.getRow());
210     }
211 
212     if (in.isSetDurability()) {
213       out.setDurability(durabilityFromThrift(in.getDurability()));
214     }
215 
216     for (TColumnValue columnValue : in.getColumnValues()) {
217       if (columnValue.isSetTimestamp()) {
218         out.addImmutable(
219             columnValue.getFamily(), columnValue.getQualifier(), columnValue.getTimestamp(),
220             columnValue.getValue());
221       } else {
222         out.addImmutable(
223             columnValue.getFamily(), columnValue.getQualifier(), columnValue.getValue());
224       }
225     }
226 
227     if (in.isSetAttributes()) {
228       addAttributes(out,in.getAttributes());
229     }
230     
231     if (in.getCellVisibility() != null) {
232       out.setCellVisibility(new CellVisibility(in.getCellVisibility().getExpression()));
233     }
234 
235     return out;
236   }
237 
238   /**
239    * Converts multiple {@link TPut}s (Thrift) into a list of {@link Put}s (HBase).
240    *
241    * @param in list of <code>TPut</code>s to convert
242    *
243    * @return list of converted <code>Put</code>s
244    *
245    * @see #putFromThrift(TPut)
246    */
247   public static List<Put> putsFromThrift(List<TPut> in) {
248     List<Put> out = new ArrayList<Put>(in.size());
249     for (TPut put : in) {
250       out.add(putFromThrift(put));
251     }
252     return out;
253   }
254 
255   /**
256    * Creates a {@link Delete} (HBase) from a {@link TDelete} (Thrift).
257    *
258    * @param in the <code>TDelete</code> to convert
259    *
260    * @return converted <code>Delete</code>
261    */
262   public static Delete deleteFromThrift(TDelete in) {
263     Delete out;
264 
265     if (in.isSetColumns()) {
266       out = new Delete(in.getRow());
267       for (TColumn column : in.getColumns()) {
268         if (column.isSetQualifier()) {
269           if (column.isSetTimestamp()) {
270             if (in.isSetDeleteType() &&
271                 in.getDeleteType().equals(TDeleteType.DELETE_COLUMNS))
272               out.addColumns(column.getFamily(), column.getQualifier(), column.getTimestamp());
273             else
274               out.addColumn(column.getFamily(), column.getQualifier(), column.getTimestamp());
275           } else {
276             if (in.isSetDeleteType() &&
277                 in.getDeleteType().equals(TDeleteType.DELETE_COLUMNS))
278               out.addColumns(column.getFamily(), column.getQualifier());
279             else
280               out.addColumn(column.getFamily(), column.getQualifier());
281           }
282 
283         } else {
284           if (column.isSetTimestamp()) {
285             out.addFamily(column.getFamily(), column.getTimestamp());
286           } else {
287             out.addFamily(column.getFamily());
288           }
289         }
290       }
291     } else {
292       if (in.isSetTimestamp()) {
293         out = new Delete(in.getRow(), in.getTimestamp());
294       } else {
295         out = new Delete(in.getRow());
296       }
297     }
298 
299     if (in.isSetAttributes()) {
300       addAttributes(out,in.getAttributes());
301     }
302 
303     if (in.isSetDurability()) {
304       out.setDurability(durabilityFromThrift(in.getDurability()));
305     }
306 
307     return out;
308   }
309 
310   /**
311    * Converts multiple {@link TDelete}s (Thrift) into a list of {@link Delete}s (HBase).
312    *
313    * @param in list of <code>TDelete</code>s to convert
314    *
315    * @return list of converted <code>Delete</code>s
316    *
317    * @see #deleteFromThrift(TDelete)
318    */
319 
320   public static List<Delete> deletesFromThrift(List<TDelete> in) {
321     List<Delete> out = new ArrayList<Delete>(in.size());
322     for (TDelete delete : in) {
323       out.add(deleteFromThrift(delete));
324     }
325     return out;
326   }
327 
328   public static TDelete deleteFromHBase(Delete in) {
329     TDelete out = new TDelete(ByteBuffer.wrap(in.getRow()));
330 
331     List<TColumn> columns = new ArrayList<TColumn>();
332     long rowTimestamp = in.getTimeStamp();
333     if (rowTimestamp != HConstants.LATEST_TIMESTAMP) {
334       out.setTimestamp(rowTimestamp);
335     }
336 
337     // Map<family, List<KeyValue>>
338     for (Map.Entry<byte[], List<org.apache.hadoop.hbase.Cell>> familyEntry:
339         in.getFamilyCellMap().entrySet()) {
340       TColumn column = new TColumn(ByteBuffer.wrap(familyEntry.getKey()));
341       for (org.apache.hadoop.hbase.Cell cell: familyEntry.getValue()) {
342         byte[] family = CellUtil.cloneFamily(cell);
343         byte[] qualifier = CellUtil.cloneQualifier(cell);
344         long timestamp = cell.getTimestamp();
345         if (family != null) {
346           column.setFamily(family);
347         }
348         if (qualifier != null) {
349           column.setQualifier(qualifier);
350         }
351         if (timestamp != HConstants.LATEST_TIMESTAMP) {
352           column.setTimestamp(timestamp);
353         }
354       }
355       columns.add(column);
356     }
357     out.setColumns(columns);
358 
359     return out;
360   }
361 
362   /**
363    * Creates a {@link RowMutations} (HBase) from a {@link TRowMutations} (Thrift)
364    *
365    * @param in the <code>TRowMutations</code> to convert
366    *
367    * @return converted <code>RowMutations</code>
368    */
369   public static RowMutations rowMutationsFromThrift(TRowMutations in) throws IOException {
370     RowMutations out = new RowMutations(in.getRow());
371     List<TMutation> mutations = in.getMutations();
372     for (TMutation mutation : mutations) {
373       if (mutation.isSetPut()) {
374         out.add(putFromThrift(mutation.getPut()));
375       }
376       if (mutation.isSetDeleteSingle()) {
377         out.add(deleteFromThrift(mutation.getDeleteSingle()));
378       }
379     }
380     return out;
381   }
382 
383   public static Scan scanFromThrift(TScan in) throws IOException {
384     Scan out = new Scan();
385 
386     if (in.isSetStartRow())
387       out.setStartRow(in.getStartRow());
388     if (in.isSetStopRow())
389       out.setStopRow(in.getStopRow());
390     if (in.isSetCaching())
391       out.setCaching(in.getCaching());
392     if (in.isSetMaxVersions()) {
393       out.setMaxVersions(in.getMaxVersions());
394     }
395 
396     if (in.isSetColumns()) {
397       for (TColumn column : in.getColumns()) {
398         if (column.isSetQualifier()) {
399           out.addColumn(column.getFamily(), column.getQualifier());
400         } else {
401           out.addFamily(column.getFamily());
402         }
403       }
404     }
405 
406     TTimeRange timeRange = in.getTimeRange();
407     if (timeRange != null &&
408         timeRange.isSetMinStamp() && timeRange.isSetMaxStamp()) {
409       out.setTimeRange(timeRange.getMinStamp(), timeRange.getMaxStamp());
410     }
411 
412     if (in.isSetBatchSize()) {
413       out.setBatch(in.getBatchSize());
414     }
415 
416     if (in.isSetFilterString()) {
417       ParseFilter parseFilter = new ParseFilter();
418       out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
419     }
420 
421     if (in.isSetAttributes()) {
422       addAttributes(out,in.getAttributes());
423     }
424     
425     if (in.isSetAuthorizations()) {
426       out.setAuthorizations(new Authorizations(in.getAuthorizations().getLabels()));
427     }
428 
429     if (in.isSetReversed()) {
430       out.setReversed(in.isReversed());
431     }
432 
433     return out;
434   }
435 
436   public static Increment incrementFromThrift(TIncrement in) throws IOException {
437     Increment out = new Increment(in.getRow());
438     for (TColumnIncrement column : in.getColumns()) {
439       out.addColumn(column.getFamily(), column.getQualifier(), column.getAmount());
440     }
441 
442     if (in.isSetAttributes()) {
443       addAttributes(out,in.getAttributes());
444     }
445 
446     if (in.isSetDurability()) {
447       out.setDurability(durabilityFromThrift(in.getDurability()));
448     }
449     
450     if(in.getCellVisibility() != null) {
451       out.setCellVisibility(new CellVisibility(in.getCellVisibility().getExpression()));
452     }
453 
454     return out;
455   }
456 
457   public static Append appendFromThrift(TAppend append) throws IOException {
458     Append out = new Append(append.getRow());
459     for (TColumnValue column : append.getColumns()) {
460       out.add(column.getFamily(), column.getQualifier(), column.getValue());
461     }
462 
463     if (append.isSetAttributes()) {
464       addAttributes(out, append.getAttributes());
465     }
466 
467     if (append.isSetDurability()) {
468       out.setDurability(durabilityFromThrift(append.getDurability()));
469     }
470     
471     if(append.getCellVisibility() != null) {
472       out.setCellVisibility(new CellVisibility(append.getCellVisibility().getExpression()));
473     }
474 
475     return out;
476   }
477 
478   public static THRegionLocation regionLocationFromHBase(HRegionLocation hrl) {
479     HRegionInfo hri = hrl.getRegionInfo();
480     ServerName serverName = hrl.getServerName();
481 
482     THRegionInfo thRegionInfo = new THRegionInfo();
483     THRegionLocation thRegionLocation = new THRegionLocation();
484     TServerName tServerName = new TServerName();
485 
486     tServerName.setHostName(serverName.getHostname());
487     tServerName.setPort(serverName.getPort());
488     tServerName.setStartCode(serverName.getStartcode());
489 
490     thRegionInfo.setTableName(hri.getTable().getName());
491     thRegionInfo.setEndKey(hri.getEndKey());
492     thRegionInfo.setStartKey(hri.getStartKey());
493     thRegionInfo.setOffline(hri.isOffline());
494     thRegionInfo.setSplit(hri.isSplit());
495     thRegionInfo.setReplicaId(hri.getReplicaId());
496 
497     thRegionLocation.setRegionInfo(thRegionInfo);
498     thRegionLocation.setServerName(tServerName);
499 
500     return thRegionLocation;
501   }
502 
503   public static List<THRegionLocation> regionLocationsFromHBase(List<HRegionLocation> locations) {
504     List<THRegionLocation> tlocations = new ArrayList<THRegionLocation>(locations.size());
505     for (HRegionLocation hrl:locations) {
506       tlocations.add(regionLocationFromHBase(hrl));
507     }
508     return tlocations;
509   }
510 
511   /**
512    * Adds all the attributes into the Operation object
513    */
514   private static void addAttributes(OperationWithAttributes op,
515                                     Map<ByteBuffer, ByteBuffer> attributes) {
516     if (attributes == null || attributes.size() == 0) {
517       return;
518     }
519     for (Map.Entry<ByteBuffer, ByteBuffer> entry : attributes.entrySet()) {
520       String name = Bytes.toStringBinary(getBytes(entry.getKey()));
521       byte[] value =  getBytes(entry.getValue());
522       op.setAttribute(name, value);
523     }
524   }
525 
526   private static Durability durabilityFromThrift(TDurability tDurability) {
527     switch (tDurability.getValue()) {
528       case 1: return Durability.SKIP_WAL;
529       case 2: return Durability.ASYNC_WAL;
530       case 3: return Durability.SYNC_WAL;
531       case 4: return Durability.FSYNC_WAL;
532       default: return null;
533     }
534   }
535 
536   public static CompareOp compareOpFromThrift(TCompareOp tCompareOp) {
537     switch (tCompareOp.getValue()) {
538       case 0: return CompareOp.LESS;
539       case 1: return CompareOp.LESS_OR_EQUAL;
540       case 2: return CompareOp.EQUAL;
541       case 3: return CompareOp.NOT_EQUAL;
542       case 4: return CompareOp.GREATER_OR_EQUAL;
543       case 5: return CompareOp.GREATER;
544       case 6: return CompareOp.NO_OP;
545       default: return null;
546     }
547   }
548 }