View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.thrift2;
20  
21  import static org.apache.hadoop.hbase.util.Bytes.getBytes;
22  
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  import java.util.ArrayList;
26  import java.util.List;
27  import java.util.Map;
28  
29  import org.apache.hadoop.hbase.HRegionInfo;
30  import org.apache.hadoop.hbase.HRegionLocation;
31  import org.apache.hadoop.hbase.ServerName;
32  import org.apache.hadoop.hbase.classification.InterfaceAudience;
33  import org.apache.hadoop.hbase.Cell;
34  import org.apache.hadoop.hbase.CellUtil;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.client.Append;
37  import org.apache.hadoop.hbase.client.Delete;
38  import org.apache.hadoop.hbase.client.Durability;
39  import org.apache.hadoop.hbase.client.Get;
40  import org.apache.hadoop.hbase.client.HRegionLocator;
41  import org.apache.hadoop.hbase.client.Increment;
42  import org.apache.hadoop.hbase.client.OperationWithAttributes;
43  import org.apache.hadoop.hbase.client.Put;
44  import org.apache.hadoop.hbase.client.Result;
45  import org.apache.hadoop.hbase.client.RowMutations;
46  import org.apache.hadoop.hbase.client.Scan;
47  import org.apache.hadoop.hbase.filter.ParseFilter;
48  import org.apache.hadoop.hbase.security.visibility.Authorizations;
49  import org.apache.hadoop.hbase.security.visibility.CellVisibility;
50  import org.apache.hadoop.hbase.thrift2.generated.TAppend;
51  import org.apache.hadoop.hbase.thrift2.generated.TColumn;
52  import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
53  import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
54  import org.apache.hadoop.hbase.thrift2.generated.TDelete;
55  import org.apache.hadoop.hbase.thrift2.generated.TDeleteType;
56  import org.apache.hadoop.hbase.thrift2.generated.TDurability;
57  import org.apache.hadoop.hbase.thrift2.generated.TGet;
58  import org.apache.hadoop.hbase.thrift2.generated.THRegionInfo;
59  import org.apache.hadoop.hbase.thrift2.generated.THRegionLocation;
60  import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
61  import org.apache.hadoop.hbase.thrift2.generated.TMutation;
62  import org.apache.hadoop.hbase.thrift2.generated.TPut;
63  import org.apache.hadoop.hbase.thrift2.generated.TResult;
64  import org.apache.hadoop.hbase.thrift2.generated.TRowMutations;
65  import org.apache.hadoop.hbase.thrift2.generated.TScan;
66  import org.apache.hadoop.hbase.thrift2.generated.TServerName;
67  import org.apache.hadoop.hbase.thrift2.generated.TTimeRange;
68  import org.apache.hadoop.hbase.util.Bytes;
69  
70  @InterfaceAudience.Private
71  public class ThriftUtilities {
72  
73    private ThriftUtilities() {
74      throw new UnsupportedOperationException("Can't initialize class");
75    }
76  
77    /**
78     * Creates a {@link Get} (HBase) from a {@link TGet} (Thrift).
79     *
80     * This ignores any timestamps set on {@link TColumn} objects.
81     *
82     * @param in the <code>TGet</code> to convert
83     *
84     * @return <code>Get</code> object
85     *
86     * @throws IOException if an invalid time range or max version parameter is given
87     */
88    public static Get getFromThrift(TGet in) throws IOException {
89      Get out = new Get(in.getRow());
90  
91      // Timestamp overwrites time range if both are set
92      if (in.isSetTimestamp()) {
93        out.setTimeStamp(in.getTimestamp());
94      } else if (in.isSetTimeRange()) {
95        out.setTimeRange(in.getTimeRange().getMinStamp(), in.getTimeRange().getMaxStamp());
96      }
97  
98      if (in.isSetMaxVersions()) {
99        out.setMaxVersions(in.getMaxVersions());
100     }
101 
102     if (in.isSetFilterString()) {
103       ParseFilter parseFilter = new ParseFilter();
104       out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
105     }
106 
107     if (in.isSetAttributes()) {
108       addAttributes(out,in.getAttributes());
109     }
110 
111     if (in.isSetAuthorizations()) {
112       out.setAuthorizations(new Authorizations(in.getAuthorizations().getLabels()));
113     }
114     
115     if (!in.isSetColumns()) {
116       return out;
117     }
118 
119     for (TColumn column : in.getColumns()) {
120       if (column.isSetQualifier()) {
121         out.addColumn(column.getFamily(), column.getQualifier());
122       } else {
123         out.addFamily(column.getFamily());
124       }
125     }
126 
127     return out;
128   }
129 
130   /**
131    * Converts multiple {@link TGet}s (Thrift) into a list of {@link Get}s (HBase).
132    *
133    * @param in list of <code>TGet</code>s to convert
134    *
135    * @return list of <code>Get</code> objects
136    *
137    * @throws IOException if an invalid time range or max version parameter is given
138    * @see #getFromThrift(TGet)
139    */
140   public static List<Get> getsFromThrift(List<TGet> in) throws IOException {
141     List<Get> out = new ArrayList<Get>(in.size());
142     for (TGet get : in) {
143       out.add(getFromThrift(get));
144     }
145     return out;
146   }
147 
148   /**
149    * Creates a {@link TResult} (Thrift) from a {@link Result} (HBase).
150    *
151    * @param in the <code>Result</code> to convert
152    *
153    * @return converted result, returns an empty result if the input is <code>null</code>
154    */
155   public static TResult resultFromHBase(Result in) {
156     Cell[] raw = in.rawCells();
157     TResult out = new TResult();
158     byte[] row = in.getRow();
159     if (row != null) {
160       out.setRow(in.getRow());
161     }
162     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
163     for (Cell kv : raw) {
164       TColumnValue col = new TColumnValue();
165       col.setFamily(CellUtil.cloneFamily(kv));
166       col.setQualifier(CellUtil.cloneQualifier(kv));
167       col.setTimestamp(kv.getTimestamp());
168       col.setValue(CellUtil.cloneValue(kv));
169       if (kv.getTagsLength() > 0) {
170         col.setTags(CellUtil.getTagArray(kv));
171       }
172       columnValues.add(col);
173     }
174     out.setColumnValues(columnValues);
175     return out;
176   }
177 
178   /**
179    * Converts multiple {@link Result}s (HBase) into a list of {@link TResult}s (Thrift).
180    *
181    * @param in array of <code>Result</code>s to convert
182    *
183    * @return list of converted <code>TResult</code>s
184    *
185    * @see #resultFromHBase(Result)
186    */
187   public static List<TResult> resultsFromHBase(Result[] in) {
188     List<TResult> out = new ArrayList<TResult>(in.length);
189     for (Result result : in) {
190       out.add(resultFromHBase(result));
191     }
192     return out;
193   }
194 
195   /**
196    * Creates a {@link Put} (HBase) from a {@link TPut} (Thrift)
197    *
198    * @param in the <code>TPut</code> to convert
199    *
200    * @return converted <code>Put</code>
201    */
202   public static Put putFromThrift(TPut in) {
203     Put out;
204 
205     if (in.isSetTimestamp()) {
206       out = new Put(in.getRow(), in.getTimestamp());
207     } else {
208       out = new Put(in.getRow());
209     }
210 
211     if (in.isSetDurability()) {
212       out.setDurability(durabilityFromThrift(in.getDurability()));
213     }
214 
215     for (TColumnValue columnValue : in.getColumnValues()) {
216       if (columnValue.isSetTimestamp()) {
217         out.addImmutable(
218             columnValue.getFamily(), columnValue.getQualifier(), columnValue.getTimestamp(),
219             columnValue.getValue());
220       } else {
221         out.addImmutable(
222             columnValue.getFamily(), columnValue.getQualifier(), columnValue.getValue());
223       }
224     }
225 
226     if (in.isSetAttributes()) {
227       addAttributes(out,in.getAttributes());
228     }
229     
230     if (in.getCellVisibility() != null) {
231       out.setCellVisibility(new CellVisibility(in.getCellVisibility().getExpression()));
232     }
233 
234     return out;
235   }
236 
237   /**
238    * Converts multiple {@link TPut}s (Thrift) into a list of {@link Put}s (HBase).
239    *
240    * @param in list of <code>TPut</code>s to convert
241    *
242    * @return list of converted <code>Put</code>s
243    *
244    * @see #putFromThrift(TPut)
245    */
246   public static List<Put> putsFromThrift(List<TPut> in) {
247     List<Put> out = new ArrayList<Put>(in.size());
248     for (TPut put : in) {
249       out.add(putFromThrift(put));
250     }
251     return out;
252   }
253 
254   /**
255    * Creates a {@link Delete} (HBase) from a {@link TDelete} (Thrift).
256    *
257    * @param in the <code>TDelete</code> to convert
258    *
259    * @return converted <code>Delete</code>
260    */
261   public static Delete deleteFromThrift(TDelete in) {
262     Delete out;
263 
264     if (in.isSetColumns()) {
265       out = new Delete(in.getRow());
266       for (TColumn column : in.getColumns()) {
267         if (column.isSetQualifier()) {
268           if (column.isSetTimestamp()) {
269             if (in.isSetDeleteType() &&
270                 in.getDeleteType().equals(TDeleteType.DELETE_COLUMNS))
271               out.deleteColumns(column.getFamily(), column.getQualifier(), column.getTimestamp());
272             else
273               out.deleteColumn(column.getFamily(), column.getQualifier(), column.getTimestamp());
274           } else {
275             if (in.isSetDeleteType() &&
276                 in.getDeleteType().equals(TDeleteType.DELETE_COLUMNS))
277               out.deleteColumns(column.getFamily(), column.getQualifier());
278             else
279               out.deleteColumn(column.getFamily(), column.getQualifier());
280           }
281 
282         } else {
283           if (column.isSetTimestamp()) {
284             out.deleteFamily(column.getFamily(), column.getTimestamp());
285           } else {
286             out.deleteFamily(column.getFamily());
287           }
288         }
289       }
290     } else {
291       if (in.isSetTimestamp()) {
292         out = new Delete(in.getRow(), in.getTimestamp());
293       } else {
294         out = new Delete(in.getRow());
295       }
296     }
297 
298     if (in.isSetAttributes()) {
299       addAttributes(out,in.getAttributes());
300     }
301 
302     if (in.isSetDurability()) {
303       out.setDurability(durabilityFromThrift(in.getDurability()));
304     }
305 
306     return out;
307   }
308 
309   /**
310    * Converts multiple {@link TDelete}s (Thrift) into a list of {@link Delete}s (HBase).
311    *
312    * @param in list of <code>TDelete</code>s to convert
313    *
314    * @return list of converted <code>Delete</code>s
315    *
316    * @see #deleteFromThrift(TDelete)
317    */
318 
319   public static List<Delete> deletesFromThrift(List<TDelete> in) {
320     List<Delete> out = new ArrayList<Delete>(in.size());
321     for (TDelete delete : in) {
322       out.add(deleteFromThrift(delete));
323     }
324     return out;
325   }
326 
327   public static TDelete deleteFromHBase(Delete in) {
328     TDelete out = new TDelete(ByteBuffer.wrap(in.getRow()));
329 
330     List<TColumn> columns = new ArrayList<TColumn>();
331     long rowTimestamp = in.getTimeStamp();
332     if (rowTimestamp != HConstants.LATEST_TIMESTAMP) {
333       out.setTimestamp(rowTimestamp);
334     }
335 
336     // Map<family, List<KeyValue>>
337     for (Map.Entry<byte[], List<org.apache.hadoop.hbase.Cell>> familyEntry:
338         in.getFamilyCellMap().entrySet()) {
339       TColumn column = new TColumn(ByteBuffer.wrap(familyEntry.getKey()));
340       for (org.apache.hadoop.hbase.Cell cell: familyEntry.getValue()) {
341         byte[] family = CellUtil.cloneFamily(cell);
342         byte[] qualifier = CellUtil.cloneQualifier(cell);
343         long timestamp = cell.getTimestamp();
344         if (family != null) {
345           column.setFamily(family);
346         }
347         if (qualifier != null) {
348           column.setQualifier(qualifier);
349         }
350         if (timestamp != HConstants.LATEST_TIMESTAMP) {
351           column.setTimestamp(timestamp);
352         }
353       }
354       columns.add(column);
355     }
356     out.setColumns(columns);
357 
358     return out;
359   }
360 
361   /**
362    * Creates a {@link RowMutations} (HBase) from a {@link TRowMutations} (Thrift)
363    *
364    * @param in the <code>TRowMutations</code> to convert
365    *
366    * @return converted <code>RowMutations</code>
367    */
368   public static RowMutations rowMutationsFromThrift(TRowMutations in) throws IOException {
369     RowMutations out = new RowMutations(in.getRow());
370     List<TMutation> mutations = in.getMutations();
371     for (TMutation mutation : mutations) {
372       if (mutation.isSetPut()) {
373         out.add(putFromThrift(mutation.getPut()));
374       }
375       if (mutation.isSetDeleteSingle()) {
376         out.add(deleteFromThrift(mutation.getDeleteSingle()));
377       }
378     }
379     return out;
380   }
381 
382   public static Scan scanFromThrift(TScan in) throws IOException {
383     Scan out = new Scan();
384 
385     if (in.isSetStartRow())
386       out.setStartRow(in.getStartRow());
387     if (in.isSetStopRow())
388       out.setStopRow(in.getStopRow());
389     if (in.isSetCaching())
390       out.setCaching(in.getCaching());
391     if (in.isSetMaxVersions()) {
392       out.setMaxVersions(in.getMaxVersions());
393     }
394 
395     if (in.isSetColumns()) {
396       for (TColumn column : in.getColumns()) {
397         if (column.isSetQualifier()) {
398           out.addColumn(column.getFamily(), column.getQualifier());
399         } else {
400           out.addFamily(column.getFamily());
401         }
402       }
403     }
404 
405     TTimeRange timeRange = in.getTimeRange();
406     if (timeRange != null &&
407         timeRange.isSetMinStamp() && timeRange.isSetMaxStamp()) {
408       out.setTimeRange(timeRange.getMinStamp(), timeRange.getMaxStamp());
409     }
410 
411     if (in.isSetBatchSize()) {
412       out.setBatch(in.getBatchSize());
413     }
414 
415     if (in.isSetFilterString()) {
416       ParseFilter parseFilter = new ParseFilter();
417       out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
418     }
419 
420     if (in.isSetAttributes()) {
421       addAttributes(out,in.getAttributes());
422     }
423     
424     if (in.isSetAuthorizations()) {
425       out.setAuthorizations(new Authorizations(in.getAuthorizations().getLabels()));
426     }
427 
428     if (in.isSetReversed()) {
429       out.setReversed(in.isReversed());
430     }
431 
432     return out;
433   }
434 
435   public static Increment incrementFromThrift(TIncrement in) throws IOException {
436     Increment out = new Increment(in.getRow());
437     for (TColumnIncrement column : in.getColumns()) {
438       out.addColumn(column.getFamily(), column.getQualifier(), column.getAmount());
439     }
440 
441     if (in.isSetAttributes()) {
442       addAttributes(out,in.getAttributes());
443     }
444 
445     if (in.isSetDurability()) {
446       out.setDurability(durabilityFromThrift(in.getDurability()));
447     }
448     
449     if(in.getCellVisibility() != null) {
450       out.setCellVisibility(new CellVisibility(in.getCellVisibility().getExpression()));
451     }
452 
453     return out;
454   }
455 
456   public static Append appendFromThrift(TAppend append) throws IOException {
457     Append out = new Append(append.getRow());
458     for (TColumnValue column : append.getColumns()) {
459       out.add(column.getFamily(), column.getQualifier(), column.getValue());
460     }
461 
462     if (append.isSetAttributes()) {
463       addAttributes(out, append.getAttributes());
464     }
465 
466     if (append.isSetDurability()) {
467       out.setDurability(durabilityFromThrift(append.getDurability()));
468     }
469     
470     if(append.getCellVisibility() != null) {
471       out.setCellVisibility(new CellVisibility(append.getCellVisibility().getExpression()));
472     }
473 
474     return out;
475   }
476 
477   public static THRegionLocation regionLocationFromHBase(HRegionLocation hrl) {
478     HRegionInfo hri = hrl.getRegionInfo();
479     ServerName serverName = hrl.getServerName();
480 
481     THRegionInfo thRegionInfo = new THRegionInfo();
482     THRegionLocation thRegionLocation = new THRegionLocation();
483     TServerName tServerName = new TServerName();
484 
485     tServerName.setHostName(serverName.getHostname());
486     tServerName.setPort(serverName.getPort());
487     tServerName.setStartCode(serverName.getStartcode());
488 
489     thRegionInfo.setTableName(hri.getTable().getName());
490     thRegionInfo.setEndKey(hri.getEndKey());
491     thRegionInfo.setStartKey(hri.getStartKey());
492     thRegionInfo.setOffline(hri.isOffline());
493     thRegionInfo.setSplit(hri.isSplit());
494     thRegionInfo.setReplicaId(hri.getReplicaId());
495 
496     thRegionLocation.setRegionInfo(thRegionInfo);
497     thRegionLocation.setServerName(tServerName);
498 
499     return thRegionLocation;
500   }
501 
502   public static List<THRegionLocation> regionLocationsFromHBase(List<HRegionLocation> locations) {
503     List<THRegionLocation> tlocations = new ArrayList<THRegionLocation>(locations.size());
504     for (HRegionLocation hrl:locations) {
505       tlocations.add(regionLocationFromHBase(hrl));
506     }
507     return tlocations;
508   }
509 
510   /**
511    * Adds all the attributes into the Operation object
512    */
513   private static void addAttributes(OperationWithAttributes op,
514                                     Map<ByteBuffer, ByteBuffer> attributes) {
515     if (attributes == null || attributes.size() == 0) {
516       return;
517     }
518     for (Map.Entry<ByteBuffer, ByteBuffer> entry : attributes.entrySet()) {
519       String name = Bytes.toStringBinary(getBytes(entry.getKey()));
520       byte[] value =  getBytes(entry.getValue());
521       op.setAttribute(name, value);
522     }
523   }
524 
525   private static Durability durabilityFromThrift(TDurability tDurability) {
526     switch (tDurability.getValue()) {
527       case 1: return Durability.SKIP_WAL;
528       case 2: return Durability.ASYNC_WAL;
529       case 3: return Durability.SYNC_WAL;
530       case 4: return Durability.FSYNC_WAL;
531       default: return null;
532     }
533   }
534 }