View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.protobuf;
19  
20  
21  import static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME;
22  
23  import java.io.ByteArrayOutputStream;
24  import java.io.IOException;
25  import java.lang.reflect.Constructor;
26  import java.lang.reflect.InvocationTargetException;
27  import java.lang.reflect.Method;
28  import java.lang.reflect.ParameterizedType;
29  import java.lang.reflect.Type;
30  import java.nio.ByteBuffer;
31  import java.util.ArrayList;
32  import java.util.Collection;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.Map.Entry;
37  import java.util.NavigableSet;
38  
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.fs.Path;
41  import org.apache.hadoop.hbase.Cell;
42  import org.apache.hadoop.hbase.CellScanner;
43  import org.apache.hadoop.hbase.CellUtil;
44  import org.apache.hadoop.hbase.DoNotRetryIOException;
45  import org.apache.hadoop.hbase.HBaseConfiguration;
46  import org.apache.hadoop.hbase.HConstants;
47  import org.apache.hadoop.hbase.HRegionInfo;
48  import org.apache.hadoop.hbase.HTableDescriptor;
49  import org.apache.hadoop.hbase.KeyValue;
50  import org.apache.hadoop.hbase.KeyValueUtil;
51  import org.apache.hadoop.hbase.NamespaceDescriptor;
52  import org.apache.hadoop.hbase.ServerName;
53  import org.apache.hadoop.hbase.TableName;
54  import org.apache.hadoop.hbase.Tag;
55  import org.apache.hadoop.hbase.client.Append;
56  import org.apache.hadoop.hbase.client.Consistency;
57  import org.apache.hadoop.hbase.client.Delete;
58  import org.apache.hadoop.hbase.client.Durability;
59  import org.apache.hadoop.hbase.client.Get;
60  import org.apache.hadoop.hbase.client.Increment;
61  import org.apache.hadoop.hbase.client.Mutation;
62  import org.apache.hadoop.hbase.client.Put;
63  import org.apache.hadoop.hbase.client.Result;
64  import org.apache.hadoop.hbase.client.Scan;
65  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
66  import org.apache.hadoop.hbase.exceptions.DeserializationException;
67  import org.apache.hadoop.hbase.filter.ByteArrayComparable;
68  import org.apache.hadoop.hbase.filter.Filter;
69  import org.apache.hadoop.hbase.io.TimeRange;
70  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
71  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService;
72  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
73  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
74  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
75  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
76  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
77  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
78  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
79  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
80  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
81  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
82  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
83  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
84  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
85  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
86  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
87  import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
88  import org.apache.hadoop.hbase.protobuf.generated.CellProtos;
89  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
90  import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
91  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
92  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
93  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
94  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Column;
95  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall;
96  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
97  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
98  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
99  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
100 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
101 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue;
102 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue.QualifierValue;
103 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.DeleteType;
104 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
105 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
106 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
107 import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
108 import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
109 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
110 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
111 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo;
112 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
113 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
114 import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
115 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
116 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
117 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService;
118 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
119 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
120 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
121 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
122 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
123 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
124 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor.EventType;
125 import org.apache.hadoop.hbase.security.access.Permission;
126 import org.apache.hadoop.hbase.security.access.TablePermission;
127 import org.apache.hadoop.hbase.security.access.UserPermission;
128 import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier;
129 import org.apache.hadoop.hbase.security.visibility.Authorizations;
130 import org.apache.hadoop.hbase.security.visibility.CellVisibility;
131 import org.apache.hadoop.hbase.util.ByteStringer;
132 import org.apache.hadoop.hbase.util.Bytes;
133 import org.apache.hadoop.hbase.util.DynamicClassLoader;
134 import org.apache.hadoop.hbase.util.ExceptionUtil;
135 import org.apache.hadoop.hbase.util.Methods;
136 import org.apache.hadoop.hbase.util.Pair;
137 import org.apache.hadoop.io.Text;
138 import org.apache.hadoop.ipc.RemoteException;
139 import org.apache.hadoop.security.token.Token;
140 
141 import com.google.common.collect.ArrayListMultimap;
142 import com.google.common.collect.ListMultimap;
143 import com.google.common.collect.Lists;
144 import com.google.protobuf.ByteString;
145 import com.google.protobuf.InvalidProtocolBufferException;
146 import com.google.protobuf.Message;
147 import com.google.protobuf.Parser;
148 import com.google.protobuf.RpcChannel;
149 import com.google.protobuf.Service;
150 import com.google.protobuf.ServiceException;
151 import com.google.protobuf.TextFormat;
152 
153 /**
154  * Protobufs utility.
155  */
156 public final class ProtobufUtil {
157 
158   private ProtobufUtil() {
159   }
160 
161   /**
162    * Primitive type to class mapping.
163    */
164   private final static Map<String, Class<?>>
165     PRIMITIVES = new HashMap<String, Class<?>>();
166 
167 
168   /**
169    * Many results are simple: no cell, exists true or false. To save on object creations,
170    *  we reuse them across calls.
171    */
172   private final static Cell[] EMPTY_CELL_ARRAY = new Cell[]{};
173   private final static Result EMPTY_RESULT = Result.create(EMPTY_CELL_ARRAY);
174   private final static Result EMPTY_RESULT_EXISTS_TRUE = Result.create(null, true);
175   private final static Result EMPTY_RESULT_EXISTS_FALSE = Result.create(null, false);
176   private final static Result EMPTY_RESULT_STALE = Result.create(EMPTY_CELL_ARRAY, null, true);
177   private final static Result EMPTY_RESULT_EXISTS_TRUE_STALE
178     = Result.create((Cell[])null, true, true);
179   private final static Result EMPTY_RESULT_EXISTS_FALSE_STALE
180     = Result.create((Cell[])null, false, true);
181 
182   private final static ClientProtos.Result EMPTY_RESULT_PB;
183   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_TRUE;
184   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_FALSE;
185   private final static ClientProtos.Result EMPTY_RESULT_PB_STALE;
186   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_TRUE_STALE;
187   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_FALSE_STALE;
188 
189 
190   static {
191     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
192 
193     builder.setExists(true);
194     builder.setAssociatedCellCount(0);
195     EMPTY_RESULT_PB_EXISTS_TRUE =  builder.build();
196 
197     builder.setStale(true);
198     EMPTY_RESULT_PB_EXISTS_TRUE_STALE = builder.build();
199     builder.clear();
200 
201     builder.setExists(false);
202     builder.setAssociatedCellCount(0);
203     EMPTY_RESULT_PB_EXISTS_FALSE =  builder.build();
204     builder.setStale(true);
205     EMPTY_RESULT_PB_EXISTS_FALSE_STALE = builder.build();
206 
207     builder.clear();
208     builder.setAssociatedCellCount(0);
209     EMPTY_RESULT_PB =  builder.build();
210     builder.setStale(true);
211     EMPTY_RESULT_PB_STALE = builder.build();
212   }
213 
214   /**
215    * Dynamic class loader to load filter/comparators
216    */
217   private final static ClassLoader CLASS_LOADER;
218 
219   static {
220     ClassLoader parent = ProtobufUtil.class.getClassLoader();
221     Configuration conf = HBaseConfiguration.create();
222     CLASS_LOADER = new DynamicClassLoader(conf, parent);
223 
224     PRIMITIVES.put(Boolean.TYPE.getName(), Boolean.TYPE);
225     PRIMITIVES.put(Byte.TYPE.getName(), Byte.TYPE);
226     PRIMITIVES.put(Character.TYPE.getName(), Character.TYPE);
227     PRIMITIVES.put(Short.TYPE.getName(), Short.TYPE);
228     PRIMITIVES.put(Integer.TYPE.getName(), Integer.TYPE);
229     PRIMITIVES.put(Long.TYPE.getName(), Long.TYPE);
230     PRIMITIVES.put(Float.TYPE.getName(), Float.TYPE);
231     PRIMITIVES.put(Double.TYPE.getName(), Double.TYPE);
232     PRIMITIVES.put(Void.TYPE.getName(), Void.TYPE);
233   }
234 
235   /**
236    * Magic we put ahead of a serialized protobuf message.
237    * For example, all znode content is protobuf messages with the below magic
238    * for preamble.
239    */
240   public static final byte [] PB_MAGIC = new byte [] {'P', 'B', 'U', 'F'};
241   private static final String PB_MAGIC_STR = Bytes.toString(PB_MAGIC);
242 
243   /**
244    * Prepend the passed bytes with four bytes of magic, {@link #PB_MAGIC}, to flag what
245    * follows as a protobuf in hbase.  Prepend these bytes to all content written to znodes, etc.
246    * @param bytes Bytes to decorate
247    * @return The passed <code>bytes</codes> with magic prepended (Creates a new
248    * byte array that is <code>bytes.length</code> plus {@link #PB_MAGIC}.length.
249    */
250   public static byte [] prependPBMagic(final byte [] bytes) {
251     return Bytes.add(PB_MAGIC, bytes);
252   }
253 
254   /**
255    * @param bytes Bytes to check.
256    * @return True if passed <code>bytes</code> has {@link #PB_MAGIC} for a prefix.
257    */
258   public static boolean isPBMagicPrefix(final byte [] bytes) {
259     return isPBMagicPrefix(bytes, 0, bytes.length);
260   }
261 
262   /**
263    * @param bytes Bytes to check.
264    * @return True if passed <code>bytes</code> has {@link #PB_MAGIC} for a prefix.
265    */
266   public static boolean isPBMagicPrefix(final byte [] bytes, int offset, int len) {
267     if (bytes == null || len < PB_MAGIC.length) return false;
268     return Bytes.compareTo(PB_MAGIC, 0, PB_MAGIC.length, bytes, offset, PB_MAGIC.length) == 0;
269   }
270 
271   /**
272    * @param bytes
273    * @throws DeserializationException if we are missing the pb magic prefix
274    */
275   public static void expectPBMagicPrefix(final byte [] bytes) throws DeserializationException {
276     if (!isPBMagicPrefix(bytes)) {
277       throw new DeserializationException("Missing pb magic " + PB_MAGIC_STR + " prefix");
278     }
279   }
280 
281   /**
282    * @return Length of {@link #PB_MAGIC}
283    */
284   public static int lengthOfPBMagic() {
285     return PB_MAGIC.length;
286   }
287 
288   /**
289    * Return the IOException thrown by the remote server wrapped in
290    * ServiceException as cause.
291    *
292    * @param se ServiceException that wraps IO exception thrown by the server
293    * @return Exception wrapped in ServiceException or
294    *   a new IOException that wraps the unexpected ServiceException.
295    */
296   public static IOException getRemoteException(ServiceException se) {
297     Throwable e = se.getCause();
298     if (e == null) {
299       return new IOException(se);
300     }
301     if (ExceptionUtil.isInterrupt(e)) {
302       return ExceptionUtil.asInterrupt(e);
303     }
304     if (e instanceof RemoteException) {
305       e = ((RemoteException) e).unwrapRemoteException();
306     }
307     return e instanceof IOException ? (IOException) e : new IOException(se);
308   }
309 
310   /**
311    * Convert a ServerName to a protocol buffer ServerName
312    *
313    * @param serverName the ServerName to convert
314    * @return the converted protocol buffer ServerName
315    * @see #toServerName(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName)
316    */
317   public static HBaseProtos.ServerName
318       toServerName(final ServerName serverName) {
319     if (serverName == null) return null;
320     HBaseProtos.ServerName.Builder builder =
321       HBaseProtos.ServerName.newBuilder();
322     builder.setHostName(serverName.getHostname());
323     if (serverName.getPort() >= 0) {
324       builder.setPort(serverName.getPort());
325     }
326     if (serverName.getStartcode() >= 0) {
327       builder.setStartCode(serverName.getStartcode());
328     }
329     return builder.build();
330   }
331 
332   /**
333    * Convert a protocol buffer ServerName to a ServerName
334    *
335    * @param proto the protocol buffer ServerName to convert
336    * @return the converted ServerName
337    */
338   public static ServerName toServerName(final HBaseProtos.ServerName proto) {
339     if (proto == null) return null;
340     String hostName = proto.getHostName();
341     long startCode = -1;
342     int port = -1;
343     if (proto.hasPort()) {
344       port = proto.getPort();
345     }
346     if (proto.hasStartCode()) {
347       startCode = proto.getStartCode();
348     }
349     return ServerName.valueOf(hostName, port, startCode);
350   }
351 
352   /**
353    * Get HTableDescriptor[] from GetTableDescriptorsResponse protobuf
354    *
355    * @param proto the GetTableDescriptorsResponse
356    * @return HTableDescriptor[]
357    */
358   public static HTableDescriptor[] getHTableDescriptorArray(GetTableDescriptorsResponse proto) {
359     if (proto == null) return null;
360 
361     HTableDescriptor[] ret = new HTableDescriptor[proto.getTableSchemaCount()];
362     for (int i = 0; i < proto.getTableSchemaCount(); ++i) {
363       ret[i] = HTableDescriptor.convert(proto.getTableSchema(i));
364     }
365     return ret;
366   }
367 
368   /**
369    * get the split keys in form "byte [][]" from a CreateTableRequest proto
370    *
371    * @param proto the CreateTableRequest
372    * @return the split keys
373    */
374   public static byte [][] getSplitKeysArray(final CreateTableRequest proto) {
375     byte [][] splitKeys = new byte[proto.getSplitKeysCount()][];
376     for (int i = 0; i < proto.getSplitKeysCount(); ++i) {
377       splitKeys[i] = proto.getSplitKeys(i).toByteArray();
378     }
379     return splitKeys;
380   }
381 
382   /**
383    * Convert a protobuf Durability into a client Durability
384    */
385   public static Durability toDurability(
386       final ClientProtos.MutationProto.Durability proto) {
387     switch(proto) {
388     case USE_DEFAULT:
389       return Durability.USE_DEFAULT;
390     case SKIP_WAL:
391       return Durability.SKIP_WAL;
392     case ASYNC_WAL:
393       return Durability.ASYNC_WAL;
394     case SYNC_WAL:
395       return Durability.SYNC_WAL;
396     case FSYNC_WAL:
397       return Durability.FSYNC_WAL;
398     default:
399       return Durability.USE_DEFAULT;
400     }
401   }
402 
403   /**
404    * Convert a client Durability into a protbuf Durability
405    */
406   public static ClientProtos.MutationProto.Durability toDurability(
407       final Durability d) {
408     switch(d) {
409     case USE_DEFAULT:
410       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
411     case SKIP_WAL:
412       return ClientProtos.MutationProto.Durability.SKIP_WAL;
413     case ASYNC_WAL:
414       return ClientProtos.MutationProto.Durability.ASYNC_WAL;
415     case SYNC_WAL:
416       return ClientProtos.MutationProto.Durability.SYNC_WAL;
417     case FSYNC_WAL:
418       return ClientProtos.MutationProto.Durability.FSYNC_WAL;
419     default:
420       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
421     }
422   }
423 
424   /**
425    * Convert a protocol buffer Get to a client Get
426    *
427    * @param proto the protocol buffer Get to convert
428    * @return the converted client Get
429    * @throws IOException
430    */
431   public static Get toGet(
432       final ClientProtos.Get proto) throws IOException {
433     if (proto == null) return null;
434     byte[] row = proto.getRow().toByteArray();
435     Get get = new Get(row);
436     if (proto.hasCacheBlocks()) {
437       get.setCacheBlocks(proto.getCacheBlocks());
438     }
439     if (proto.hasMaxVersions()) {
440       get.setMaxVersions(proto.getMaxVersions());
441     }
442     if (proto.hasStoreLimit()) {
443       get.setMaxResultsPerColumnFamily(proto.getStoreLimit());
444     }
445     if (proto.hasStoreOffset()) {
446       get.setRowOffsetPerColumnFamily(proto.getStoreOffset());
447     }
448     if (proto.hasTimeRange()) {
449       HBaseProtos.TimeRange timeRange = proto.getTimeRange();
450       long minStamp = 0;
451       long maxStamp = Long.MAX_VALUE;
452       if (timeRange.hasFrom()) {
453         minStamp = timeRange.getFrom();
454       }
455       if (timeRange.hasTo()) {
456         maxStamp = timeRange.getTo();
457       }
458       get.setTimeRange(minStamp, maxStamp);
459     }
460     if (proto.hasFilter()) {
461       FilterProtos.Filter filter = proto.getFilter();
462       get.setFilter(ProtobufUtil.toFilter(filter));
463     }
464     for (NameBytesPair attribute: proto.getAttributeList()) {
465       get.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
466     }
467     if (proto.getColumnCount() > 0) {
468       for (Column column: proto.getColumnList()) {
469         byte[] family = column.getFamily().toByteArray();
470         if (column.getQualifierCount() > 0) {
471           for (ByteString qualifier: column.getQualifierList()) {
472             get.addColumn(family, qualifier.toByteArray());
473           }
474         } else {
475           get.addFamily(family);
476         }
477       }
478     }
479     if (proto.hasExistenceOnly() && proto.getExistenceOnly()){
480       get.setCheckExistenceOnly(true);
481     }
482     if (proto.hasClosestRowBefore() && proto.getClosestRowBefore()){
483       get.setClosestRowBefore(true);
484     }
485     if (proto.hasConsistency()) {
486       get.setConsistency(toConsistency(proto.getConsistency()));
487     }
488     return get;
489   }
490 
491   public static Consistency toConsistency(ClientProtos.Consistency consistency) {
492     switch (consistency) {
493       case STRONG : return Consistency.STRONG;
494       case TIMELINE : return Consistency.TIMELINE;
495       default : return Consistency.STRONG;
496     }
497   }
498 
499   public static ClientProtos.Consistency toConsistency(Consistency consistency) {
500     switch (consistency) {
501       case STRONG : return ClientProtos.Consistency.STRONG;
502       case TIMELINE : return ClientProtos.Consistency.TIMELINE;
503       default : return ClientProtos.Consistency.STRONG;
504     }
505   }
506 
507   /**
508    * Convert a protocol buffer Mutate to a Put.
509    *
510    * @param proto The protocol buffer MutationProto to convert
511    * @return A client Put.
512    * @throws IOException
513    */
514   public static Put toPut(final MutationProto proto)
515   throws IOException {
516     return toPut(proto, null);
517   }
518 
519   /**
520    * Convert a protocol buffer Mutate to a Put.
521    *
522    * @param proto The protocol buffer MutationProto to convert
523    * @param cellScanner If non-null, the Cell data that goes with this proto.
524    * @return A client Put.
525    * @throws IOException
526    */
527   public static Put toPut(final MutationProto proto, final CellScanner cellScanner)
528   throws IOException {
529     // TODO: Server-side at least why do we convert back to the Client types?  Why not just pb it?
530     MutationType type = proto.getMutateType();
531     assert type == MutationType.PUT: type.name();
532     long timestamp = proto.hasTimestamp()? proto.getTimestamp(): HConstants.LATEST_TIMESTAMP;
533     Put put = null;
534     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
535     if (cellCount > 0) {
536       // The proto has metadata only and the data is separate to be found in the cellScanner.
537       if (cellScanner == null) {
538         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
539             toShortString(proto));
540       }
541       for (int i = 0; i < cellCount; i++) {
542         if (!cellScanner.advance()) {
543           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
544             " no cell returned: " + toShortString(proto));
545         }
546         Cell cell = cellScanner.current();
547         if (put == null) {
548           put = new Put(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
549         }
550         put.add(cell);
551       }
552     } else {
553       if (proto.hasRow()) {
554         put = new Put(proto.getRow().asReadOnlyByteBuffer(), timestamp);
555       } else {
556         throw new IllegalArgumentException("row cannot be null");
557       }
558       // The proto has the metadata and the data itself
559       for (ColumnValue column: proto.getColumnValueList()) {
560         byte[] family = column.getFamily().toByteArray();
561         for (QualifierValue qv: column.getQualifierValueList()) {
562           if (!qv.hasValue()) {
563             throw new DoNotRetryIOException(
564                 "Missing required field: qualifier value");
565           }
566           ByteBuffer qualifier =
567               qv.hasQualifier() ? qv.getQualifier().asReadOnlyByteBuffer() : null;
568           ByteBuffer value =
569               qv.hasValue() ? qv.getValue().asReadOnlyByteBuffer() : null;
570           long ts = timestamp;
571           if (qv.hasTimestamp()) {
572             ts = qv.getTimestamp();
573           }
574           byte[] tags;
575           if (qv.hasTags()) {
576             tags = qv.getTags().toByteArray();
577             Object[] array = Tag.asList(tags, 0, (short)tags.length).toArray();
578             Tag[] tagArray = new Tag[array.length];
579             for(int i = 0; i< array.length; i++) {
580               tagArray[i] = (Tag)array[i];
581             }
582             put.addImmutable(family, qualifier, ts, value, tagArray);
583           } else {
584             put.addImmutable(family, qualifier, ts, value);
585           }
586         }
587       }
588     }
589     put.setDurability(toDurability(proto.getDurability()));
590     for (NameBytesPair attribute: proto.getAttributeList()) {
591       put.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
592     }
593     return put;
594   }
595 
596   /**
597    * Convert a protocol buffer Mutate to a Delete
598    *
599    * @param proto the protocol buffer Mutate to convert
600    * @return the converted client Delete
601    * @throws IOException
602    */
603   public static Delete toDelete(final MutationProto proto)
604   throws IOException {
605     return toDelete(proto, null);
606   }
607 
608   /**
609    * Convert a protocol buffer Mutate to a Delete
610    *
611    * @param proto the protocol buffer Mutate to convert
612    * @param cellScanner if non-null, the data that goes with this delete.
613    * @return the converted client Delete
614    * @throws IOException
615    */
616   public static Delete toDelete(final MutationProto proto, final CellScanner cellScanner)
617   throws IOException {
618     MutationType type = proto.getMutateType();
619     assert type == MutationType.DELETE : type.name();
620     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
621     long timestamp = HConstants.LATEST_TIMESTAMP;
622     if (proto.hasTimestamp()) {
623       timestamp = proto.getTimestamp();
624     }
625     Delete delete = null;
626     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
627     if (cellCount > 0) {
628       // The proto has metadata only and the data is separate to be found in the cellScanner.
629       if (cellScanner == null) {
630         // TextFormat should be fine for a Delete since it carries no data, just coordinates.
631         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
632           TextFormat.shortDebugString(proto));
633       }
634       for (int i = 0; i < cellCount; i++) {
635         if (!cellScanner.advance()) {
636           // TextFormat should be fine for a Delete since it carries no data, just coordinates.
637           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
638             " no cell returned: " + TextFormat.shortDebugString(proto));
639         }
640         Cell cell = cellScanner.current();
641         if (delete == null) {
642           delete =
643             new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
644         }
645         delete.addDeleteMarker(KeyValueUtil.ensureKeyValue(cell));
646       }
647     } else {
648       delete = new Delete(row, timestamp);
649       for (ColumnValue column: proto.getColumnValueList()) {
650         byte[] family = column.getFamily().toByteArray();
651         for (QualifierValue qv: column.getQualifierValueList()) {
652           DeleteType deleteType = qv.getDeleteType();
653           byte[] qualifier = null;
654           if (qv.hasQualifier()) {
655             qualifier = qv.getQualifier().toByteArray();
656           }
657           long ts = HConstants.LATEST_TIMESTAMP;
658           if (qv.hasTimestamp()) {
659             ts = qv.getTimestamp();
660           }
661           if (deleteType == DeleteType.DELETE_ONE_VERSION) {
662             delete.deleteColumn(family, qualifier, ts);
663           } else if (deleteType == DeleteType.DELETE_MULTIPLE_VERSIONS) {
664             delete.deleteColumns(family, qualifier, ts);
665           } else if (deleteType == DeleteType.DELETE_FAMILY_VERSION) {
666             delete.deleteFamilyVersion(family, ts);
667           } else {
668             delete.deleteFamily(family, ts);
669           }
670         }
671       }
672     }
673     delete.setDurability(toDurability(proto.getDurability()));
674     for (NameBytesPair attribute: proto.getAttributeList()) {
675       delete.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
676     }
677     return delete;
678   }
679 
680   /**
681    * Convert a protocol buffer Mutate to an Append
682    * @param cellScanner
683    * @param proto the protocol buffer Mutate to convert
684    * @return the converted client Append
685    * @throws IOException
686    */
687   public static Append toAppend(final MutationProto proto, final CellScanner cellScanner)
688   throws IOException {
689     MutationType type = proto.getMutateType();
690     assert type == MutationType.APPEND : type.name();
691     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
692     Append append = null;
693     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
694     if (cellCount > 0) {
695       // The proto has metadata only and the data is separate to be found in the cellScanner.
696       if (cellScanner == null) {
697         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
698           toShortString(proto));
699       }
700       for (int i = 0; i < cellCount; i++) {
701         if (!cellScanner.advance()) {
702           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
703             " no cell returned: " + toShortString(proto));
704         }
705         Cell cell = cellScanner.current();
706         if (append == null) {
707           append = new Append(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
708         }
709         append.add(KeyValueUtil.ensureKeyValue(cell));
710       }
711     } else {
712       append = new Append(row);
713       for (ColumnValue column: proto.getColumnValueList()) {
714         byte[] family = column.getFamily().toByteArray();
715         for (QualifierValue qv: column.getQualifierValueList()) {
716           byte[] qualifier = qv.getQualifier().toByteArray();
717           if (!qv.hasValue()) {
718             throw new DoNotRetryIOException(
719               "Missing required field: qualifier value");
720           }
721           byte[] value = qv.getValue().toByteArray();
722           byte[] tags = null;
723           if (qv.hasTags()) {
724             tags = qv.getTags().toByteArray();
725           }
726           append.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
727               KeyValue.Type.Put, value, tags));
728         }
729       }
730     }
731     append.setDurability(toDurability(proto.getDurability()));
732     for (NameBytesPair attribute: proto.getAttributeList()) {
733       append.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
734     }
735     return append;
736   }
737 
738   /**
739    * Convert a MutateRequest to Mutation
740    *
741    * @param proto the protocol buffer Mutate to convert
742    * @return the converted Mutation
743    * @throws IOException
744    */
745   public static Mutation toMutation(final MutationProto proto) throws IOException {
746     MutationType type = proto.getMutateType();
747     if (type == MutationType.APPEND) {
748       return toAppend(proto, null);
749     }
750     if (type == MutationType.DELETE) {
751       return toDelete(proto, null);
752     }
753     if (type == MutationType.PUT) {
754       return toPut(proto, null);
755     }
756     throw new IOException("Unknown mutation type " + type);
757   }
758 
759   /**
760    * Convert a protocol buffer Mutate to an Increment
761    *
762    * @param proto the protocol buffer Mutate to convert
763    * @return the converted client Increment
764    * @throws IOException
765    */
766   public static Increment toIncrement(final MutationProto proto, final CellScanner cellScanner)
767   throws IOException {
768     MutationType type = proto.getMutateType();
769     assert type == MutationType.INCREMENT : type.name();
770     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
771     Increment increment = null;
772     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
773     if (cellCount > 0) {
774       // The proto has metadata only and the data is separate to be found in the cellScanner.
775       if (cellScanner == null) {
776         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
777           TextFormat.shortDebugString(proto));
778       }
779       for (int i = 0; i < cellCount; i++) {
780         if (!cellScanner.advance()) {
781           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
782             " no cell returned: " + TextFormat.shortDebugString(proto));
783         }
784         Cell cell = cellScanner.current();
785         if (increment == null) {
786           increment = new Increment(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
787         }
788         increment.add(KeyValueUtil.ensureKeyValue(cell));
789       }
790     } else {
791       increment = new Increment(row);
792       for (ColumnValue column: proto.getColumnValueList()) {
793         byte[] family = column.getFamily().toByteArray();
794         for (QualifierValue qv: column.getQualifierValueList()) {
795           byte[] qualifier = qv.getQualifier().toByteArray();
796           if (!qv.hasValue()) {
797             throw new DoNotRetryIOException("Missing required field: qualifier value");
798           }
799           byte[] value = qv.getValue().toByteArray();
800           byte[] tags = null;
801           if (qv.hasTags()) {
802             tags = qv.getTags().toByteArray();
803           }
804           increment.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
805               KeyValue.Type.Put, value, tags));
806         }
807       }
808     }
809     if (proto.hasTimeRange()) {
810       HBaseProtos.TimeRange timeRange = proto.getTimeRange();
811       long minStamp = 0;
812       long maxStamp = Long.MAX_VALUE;
813       if (timeRange.hasFrom()) {
814         minStamp = timeRange.getFrom();
815       }
816       if (timeRange.hasTo()) {
817         maxStamp = timeRange.getTo();
818       }
819       increment.setTimeRange(minStamp, maxStamp);
820     }
821     increment.setDurability(toDurability(proto.getDurability()));
822     for (NameBytesPair attribute : proto.getAttributeList()) {
823       increment.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
824     }
825     return increment;
826   }
827 
828   /**
829    * Convert a client Scan to a protocol buffer Scan
830    *
831    * @param scan the client Scan to convert
832    * @return the converted protocol buffer Scan
833    * @throws IOException
834    */
835   public static ClientProtos.Scan toScan(
836       final Scan scan) throws IOException {
837     ClientProtos.Scan.Builder scanBuilder =
838       ClientProtos.Scan.newBuilder();
839     scanBuilder.setCacheBlocks(scan.getCacheBlocks());
840     if (scan.getBatch() > 0) {
841       scanBuilder.setBatchSize(scan.getBatch());
842     }
843     if (scan.getMaxResultSize() > 0) {
844       scanBuilder.setMaxResultSize(scan.getMaxResultSize());
845     }
846     if (scan.isSmall()) {
847       scanBuilder.setSmall(scan.isSmall());
848     }
849     Boolean loadColumnFamiliesOnDemand = scan.getLoadColumnFamiliesOnDemandValue();
850     if (loadColumnFamiliesOnDemand != null) {
851       scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand.booleanValue());
852     }
853     scanBuilder.setMaxVersions(scan.getMaxVersions());
854     TimeRange timeRange = scan.getTimeRange();
855     if (!timeRange.isAllTime()) {
856       HBaseProtos.TimeRange.Builder timeRangeBuilder =
857         HBaseProtos.TimeRange.newBuilder();
858       timeRangeBuilder.setFrom(timeRange.getMin());
859       timeRangeBuilder.setTo(timeRange.getMax());
860       scanBuilder.setTimeRange(timeRangeBuilder.build());
861     }
862     Map<String, byte[]> attributes = scan.getAttributesMap();
863     if (!attributes.isEmpty()) {
864       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
865       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
866         attributeBuilder.setName(attribute.getKey());
867         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
868         scanBuilder.addAttribute(attributeBuilder.build());
869       }
870     }
871     byte[] startRow = scan.getStartRow();
872     if (startRow != null && startRow.length > 0) {
873       scanBuilder.setStartRow(ByteStringer.wrap(startRow));
874     }
875     byte[] stopRow = scan.getStopRow();
876     if (stopRow != null && stopRow.length > 0) {
877       scanBuilder.setStopRow(ByteStringer.wrap(stopRow));
878     }
879     if (scan.hasFilter()) {
880       scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
881     }
882     if (scan.hasFamilies()) {
883       Column.Builder columnBuilder = Column.newBuilder();
884       for (Map.Entry<byte[],NavigableSet<byte []>>
885           family: scan.getFamilyMap().entrySet()) {
886         columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
887         NavigableSet<byte []> qualifiers = family.getValue();
888         columnBuilder.clearQualifier();
889         if (qualifiers != null && qualifiers.size() > 0) {
890           for (byte [] qualifier: qualifiers) {
891             columnBuilder.addQualifier(ByteStringer.wrap(qualifier));
892           }
893         }
894         scanBuilder.addColumn(columnBuilder.build());
895       }
896     }
897     if (scan.getMaxResultsPerColumnFamily() >= 0) {
898       scanBuilder.setStoreLimit(scan.getMaxResultsPerColumnFamily());
899     }
900     if (scan.getRowOffsetPerColumnFamily() > 0) {
901       scanBuilder.setStoreOffset(scan.getRowOffsetPerColumnFamily());
902     }
903     if (scan.isReversed()) {
904       scanBuilder.setReversed(scan.isReversed());
905     }
906     if (scan.getConsistency() == Consistency.TIMELINE) {
907       scanBuilder.setConsistency(toConsistency(scan.getConsistency()));
908     }
909     if (scan.getCaching() > 0) {
910       scanBuilder.setCaching(scan.getCaching());
911     }
912     return scanBuilder.build();
913   }
914 
915   /**
916    * Convert a protocol buffer Scan to a client Scan
917    *
918    * @param proto the protocol buffer Scan to convert
919    * @return the converted client Scan
920    * @throws IOException
921    */
922   public static Scan toScan(
923       final ClientProtos.Scan proto) throws IOException {
924     byte [] startRow = HConstants.EMPTY_START_ROW;
925     byte [] stopRow  = HConstants.EMPTY_END_ROW;
926     if (proto.hasStartRow()) {
927       startRow = proto.getStartRow().toByteArray();
928     }
929     if (proto.hasStopRow()) {
930       stopRow = proto.getStopRow().toByteArray();
931     }
932     Scan scan = new Scan(startRow, stopRow);
933     if (proto.hasCacheBlocks()) {
934       scan.setCacheBlocks(proto.getCacheBlocks());
935     }
936     if (proto.hasMaxVersions()) {
937       scan.setMaxVersions(proto.getMaxVersions());
938     }
939     if (proto.hasStoreLimit()) {
940       scan.setMaxResultsPerColumnFamily(proto.getStoreLimit());
941     }
942     if (proto.hasStoreOffset()) {
943       scan.setRowOffsetPerColumnFamily(proto.getStoreOffset());
944     }
945     if (proto.hasLoadColumnFamiliesOnDemand()) {
946       scan.setLoadColumnFamiliesOnDemand(proto.getLoadColumnFamiliesOnDemand());
947     }
948     if (proto.hasTimeRange()) {
949       HBaseProtos.TimeRange timeRange = proto.getTimeRange();
950       long minStamp = 0;
951       long maxStamp = Long.MAX_VALUE;
952       if (timeRange.hasFrom()) {
953         minStamp = timeRange.getFrom();
954       }
955       if (timeRange.hasTo()) {
956         maxStamp = timeRange.getTo();
957       }
958       scan.setTimeRange(minStamp, maxStamp);
959     }
960     if (proto.hasFilter()) {
961       FilterProtos.Filter filter = proto.getFilter();
962       scan.setFilter(ProtobufUtil.toFilter(filter));
963     }
964     if (proto.hasBatchSize()) {
965       scan.setBatch(proto.getBatchSize());
966     }
967     if (proto.hasMaxResultSize()) {
968       scan.setMaxResultSize(proto.getMaxResultSize());
969     }
970     if (proto.hasSmall()) {
971       scan.setSmall(proto.getSmall());
972     }
973     for (NameBytesPair attribute: proto.getAttributeList()) {
974       scan.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
975     }
976     if (proto.getColumnCount() > 0) {
977       for (Column column: proto.getColumnList()) {
978         byte[] family = column.getFamily().toByteArray();
979         if (column.getQualifierCount() > 0) {
980           for (ByteString qualifier: column.getQualifierList()) {
981             scan.addColumn(family, qualifier.toByteArray());
982           }
983         } else {
984           scan.addFamily(family);
985         }
986       }
987     }
988     if (proto.hasReversed()) {
989       scan.setReversed(proto.getReversed());
990     }
991     if (proto.hasConsistency()) {
992       scan.setConsistency(toConsistency(proto.getConsistency()));
993     }
994     if (proto.hasCaching()) {
995       scan.setCaching(proto.getCaching());
996     }
997     return scan;
998   }
999 
1000   /**
1001    * Create a protocol buffer Get based on a client Get.
1002    *
1003    * @param get the client Get
1004    * @return a protocol buffer Get
1005    * @throws IOException
1006    */
1007   public static ClientProtos.Get toGet(
1008       final Get get) throws IOException {
1009     ClientProtos.Get.Builder builder =
1010       ClientProtos.Get.newBuilder();
1011     builder.setRow(ByteStringer.wrap(get.getRow()));
1012     builder.setCacheBlocks(get.getCacheBlocks());
1013     builder.setMaxVersions(get.getMaxVersions());
1014     if (get.getFilter() != null) {
1015       builder.setFilter(ProtobufUtil.toFilter(get.getFilter()));
1016     }
1017     TimeRange timeRange = get.getTimeRange();
1018     if (!timeRange.isAllTime()) {
1019       HBaseProtos.TimeRange.Builder timeRangeBuilder =
1020         HBaseProtos.TimeRange.newBuilder();
1021       timeRangeBuilder.setFrom(timeRange.getMin());
1022       timeRangeBuilder.setTo(timeRange.getMax());
1023       builder.setTimeRange(timeRangeBuilder.build());
1024     }
1025     Map<String, byte[]> attributes = get.getAttributesMap();
1026     if (!attributes.isEmpty()) {
1027       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1028       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
1029         attributeBuilder.setName(attribute.getKey());
1030         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1031         builder.addAttribute(attributeBuilder.build());
1032       }
1033     }
1034     if (get.hasFamilies()) {
1035       Column.Builder columnBuilder = Column.newBuilder();
1036       Map<byte[], NavigableSet<byte[]>> families = get.getFamilyMap();
1037       for (Map.Entry<byte[], NavigableSet<byte[]>> family: families.entrySet()) {
1038         NavigableSet<byte[]> qualifiers = family.getValue();
1039         columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1040         columnBuilder.clearQualifier();
1041         if (qualifiers != null && qualifiers.size() > 0) {
1042           for (byte[] qualifier: qualifiers) {
1043             columnBuilder.addQualifier(ByteStringer.wrap(qualifier));
1044           }
1045         }
1046         builder.addColumn(columnBuilder.build());
1047       }
1048     }
1049     if (get.getMaxResultsPerColumnFamily() >= 0) {
1050       builder.setStoreLimit(get.getMaxResultsPerColumnFamily());
1051     }
1052     if (get.getRowOffsetPerColumnFamily() > 0) {
1053       builder.setStoreOffset(get.getRowOffsetPerColumnFamily());
1054     }
1055     if (get.isCheckExistenceOnly()){
1056       builder.setExistenceOnly(true);
1057     }
1058     if (get.isClosestRowBefore()){
1059       builder.setClosestRowBefore(true);
1060     }
1061     if (get.getConsistency() != null && get.getConsistency() != Consistency.STRONG) {
1062       builder.setConsistency(toConsistency(get.getConsistency()));
1063     }
1064 
1065     return builder.build();
1066   }
1067 
1068   /**
1069    * Convert a client Increment to a protobuf Mutate.
1070    *
1071    * @param increment
1072    * @return the converted mutate
1073    */
1074   public static MutationProto toMutation(
1075     final Increment increment, final MutationProto.Builder builder, long nonce) {
1076     builder.setRow(ByteStringer.wrap(increment.getRow()));
1077     builder.setMutateType(MutationType.INCREMENT);
1078     builder.setDurability(toDurability(increment.getDurability()));
1079     if (nonce != HConstants.NO_NONCE) {
1080       builder.setNonce(nonce);
1081     }
1082     TimeRange timeRange = increment.getTimeRange();
1083     if (!timeRange.isAllTime()) {
1084       HBaseProtos.TimeRange.Builder timeRangeBuilder =
1085         HBaseProtos.TimeRange.newBuilder();
1086       timeRangeBuilder.setFrom(timeRange.getMin());
1087       timeRangeBuilder.setTo(timeRange.getMax());
1088       builder.setTimeRange(timeRangeBuilder.build());
1089     }
1090     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1091     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1092     for (Map.Entry<byte[], List<Cell>> family: increment.getFamilyCellMap().entrySet()) {
1093       columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1094       columnBuilder.clearQualifierValue();
1095       List<Cell> values = family.getValue();
1096       if (values != null && values.size() > 0) {
1097         for (Cell cell: values) {
1098           KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
1099           valueBuilder.setQualifier(ByteStringer.wrap(
1100               kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()));
1101           valueBuilder.setValue(ByteStringer.wrap(
1102               kv.getValueArray(), kv.getValueOffset(), kv.getValueLength()));
1103           if (kv.getTagsLength() > 0) {
1104             valueBuilder.setTags(ByteStringer.wrap(kv.getTagsArray(),
1105                 kv.getTagsOffset(), kv.getTagsLength()));
1106           }
1107           columnBuilder.addQualifierValue(valueBuilder.build());
1108         }
1109       }
1110       builder.addColumnValue(columnBuilder.build());
1111     }
1112     Map<String, byte[]> attributes = increment.getAttributesMap();
1113     if (!attributes.isEmpty()) {
1114       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1115       for (Map.Entry<String, byte[]> attribute : attributes.entrySet()) {
1116         attributeBuilder.setName(attribute.getKey());
1117         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1118         builder.addAttribute(attributeBuilder.build());
1119       }
1120     }
1121     return builder.build();
1122   }
1123 
1124   public static MutationProto toMutation(final MutationType type, final Mutation mutation)
1125     throws IOException {
1126     return toMutation(type, mutation, HConstants.NO_NONCE);
1127   }
1128 
1129   /**
1130    * Create a protocol buffer Mutate based on a client Mutation
1131    *
1132    * @param type
1133    * @param mutation
1134    * @return a protobuf'd Mutation
1135    * @throws IOException
1136    */
1137   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1138     final long nonce) throws IOException {
1139     return toMutation(type, mutation, MutationProto.newBuilder(), nonce);
1140   }
1141 
1142   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1143       MutationProto.Builder builder) throws IOException {
1144     return toMutation(type, mutation, builder, HConstants.NO_NONCE);
1145   }
1146 
1147   @SuppressWarnings("deprecation")
1148   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1149       MutationProto.Builder builder, long nonce)
1150   throws IOException {
1151     builder = getMutationBuilderAndSetCommonFields(type, mutation, builder);
1152     if (nonce != HConstants.NO_NONCE) {
1153       builder.setNonce(nonce);
1154     }
1155     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1156     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1157     for (Map.Entry<byte[],List<Cell>> family: mutation.getFamilyCellMap().entrySet()) {
1158       columnBuilder.clear();
1159       columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1160       for (Cell cell: family.getValue()) {
1161         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
1162         valueBuilder.setQualifier(ByteStringer.wrap(
1163             kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()));
1164         valueBuilder.setValue(ByteStringer.wrap(
1165             kv.getValueArray(), kv.getValueOffset(), kv.getValueLength()));
1166         valueBuilder.setTimestamp(kv.getTimestamp());
1167         if(cell.getTagsLength() > 0) {
1168           valueBuilder.setTags(ByteStringer.wrap(kv.getTagsArray(), kv.getTagsOffset(),
1169               kv.getTagsLength()));
1170         }
1171         if (type == MutationType.DELETE) {
1172           KeyValue.Type keyValueType = KeyValue.Type.codeToType(kv.getType());
1173           valueBuilder.setDeleteType(toDeleteType(keyValueType));
1174         }
1175         columnBuilder.addQualifierValue(valueBuilder.build());
1176       }
1177       builder.addColumnValue(columnBuilder.build());
1178     }
1179     return builder.build();
1180   }
1181 
1182   /**
1183    * Create a protocol buffer MutationProto based on a client Mutation. Does NOT include data.
1184    * Understanding is that the Cell will be transported other than via protobuf.
1185    * @param type
1186    * @param mutation
1187    * @param builder
1188    * @return a protobuf'd Mutation
1189    * @throws IOException
1190    */
1191   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1192       final MutationProto.Builder builder)  throws IOException {
1193     return toMutationNoData(type, mutation, builder, HConstants.NO_NONCE);
1194   }
1195 
1196   /**
1197    * Create a protocol buffer MutationProto based on a client Mutation.  Does NOT include data.
1198    * Understanding is that the Cell will be transported other than via protobuf.
1199    * @param type
1200    * @param mutation
1201    * @return a protobuf'd Mutation
1202    * @throws IOException
1203    */
1204   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation)
1205   throws IOException {
1206     MutationProto.Builder builder =  MutationProto.newBuilder();
1207     return toMutationNoData(type, mutation, builder);
1208   }
1209 
1210   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1211       final MutationProto.Builder builder, long nonce) throws IOException {
1212     getMutationBuilderAndSetCommonFields(type, mutation, builder);
1213     builder.setAssociatedCellCount(mutation.size());
1214     if (nonce != HConstants.NO_NONCE) {
1215       builder.setNonce(nonce);
1216     }
1217     return builder.build();
1218   }
1219 
1220   /**
1221    * Code shared by {@link #toMutation(MutationType, Mutation)} and
1222    * {@link #toMutationNoData(MutationType, Mutation)}
1223    * @param type
1224    * @param mutation
1225    * @return A partly-filled out protobuf'd Mutation.
1226    */
1227   private static MutationProto.Builder getMutationBuilderAndSetCommonFields(final MutationType type,
1228       final Mutation mutation, MutationProto.Builder builder) {
1229     builder.setRow(ByteStringer.wrap(mutation.getRow()));
1230     builder.setMutateType(type);
1231     builder.setDurability(toDurability(mutation.getDurability()));
1232     builder.setTimestamp(mutation.getTimeStamp());
1233     Map<String, byte[]> attributes = mutation.getAttributesMap();
1234     if (!attributes.isEmpty()) {
1235       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1236       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
1237         attributeBuilder.setName(attribute.getKey());
1238         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1239         builder.addAttribute(attributeBuilder.build());
1240       }
1241     }
1242     return builder;
1243   }
1244 
1245   /**
1246    * Convert a client Result to a protocol buffer Result
1247    *
1248    * @param result the client Result to convert
1249    * @return the converted protocol buffer Result
1250    */
1251   public static ClientProtos.Result toResult(final Result result) {
1252     if (result.getExists() != null) {
1253       return toResult(result.getExists(), result.isStale());
1254     }
1255 
1256     Cell[] cells = result.rawCells();
1257     if (cells == null || cells.length == 0) {
1258       return result.isStale() ? EMPTY_RESULT_PB_STALE : EMPTY_RESULT_PB;
1259     }
1260 
1261     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1262     for (Cell c : cells) {
1263       builder.addCell(toCell(c));
1264     }
1265 
1266     builder.setStale(result.isStale());
1267 
1268     return builder.build();
1269   }
1270 
1271   /**
1272    * Convert a client Result to a protocol buffer Result
1273    *
1274    * @param existence the client existence to send
1275    * @return the converted protocol buffer Result
1276    */
1277   public static ClientProtos.Result toResult(final boolean existence, boolean stale) {
1278     if (stale){
1279       return existence ? EMPTY_RESULT_PB_EXISTS_TRUE_STALE : EMPTY_RESULT_PB_EXISTS_FALSE_STALE;
1280     } else {
1281       return existence ? EMPTY_RESULT_PB_EXISTS_TRUE : EMPTY_RESULT_PB_EXISTS_FALSE;
1282     }
1283   }
1284 
1285   /**
1286    * Convert a client Result to a protocol buffer Result.
1287    * The pb Result does not include the Cell data.  That is for transport otherwise.
1288    *
1289    * @param result the client Result to convert
1290    * @return the converted protocol buffer Result
1291    */
1292   public static ClientProtos.Result toResultNoData(final Result result) {
1293     if (result.getExists() != null) return toResult(result.getExists(), result.isStale());
1294     int size = result.size();
1295     if (size == 0) return result.isStale() ? EMPTY_RESULT_PB_STALE : EMPTY_RESULT_PB;
1296     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1297     builder.setAssociatedCellCount(size);
1298     builder.setStale(result.isStale());
1299     return builder.build();
1300   }
1301 
1302   /**
1303    * Convert a protocol buffer Result to a client Result
1304    *
1305    * @param proto the protocol buffer Result to convert
1306    * @return the converted client Result
1307    */
1308   public static Result toResult(final ClientProtos.Result proto) {
1309     if (proto.hasExists()) {
1310       if (proto.getStale()) {
1311         return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE_STALE :EMPTY_RESULT_EXISTS_FALSE_STALE;
1312       }
1313       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1314     }
1315 
1316     List<CellProtos.Cell> values = proto.getCellList();
1317     if (values.isEmpty()){
1318       return proto.getStale() ? EMPTY_RESULT_STALE : EMPTY_RESULT;
1319     }
1320 
1321     List<Cell> cells = new ArrayList<Cell>(values.size());
1322     for (CellProtos.Cell c : values) {
1323       cells.add(toCell(c));
1324     }
1325     return Result.create(cells, null, proto.getStale());
1326   }
1327 
1328   /**
1329    * Convert a protocol buffer Result to a client Result
1330    *
1331    * @param proto the protocol buffer Result to convert
1332    * @param scanner Optional cell scanner.
1333    * @return the converted client Result
1334    * @throws IOException
1335    */
1336   public static Result toResult(final ClientProtos.Result proto, final CellScanner scanner)
1337   throws IOException {
1338     List<CellProtos.Cell> values = proto.getCellList();
1339 
1340     if (proto.hasExists()) {
1341       if ((values != null && !values.isEmpty()) ||
1342           (proto.hasAssociatedCellCount() && proto.getAssociatedCellCount() > 0)) {
1343         throw new IllegalArgumentException("bad proto: exists with cells is no allowed " + proto);
1344       }
1345       if (proto.getStale()) {
1346         return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE_STALE :EMPTY_RESULT_EXISTS_FALSE_STALE;
1347       }
1348       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1349     }
1350 
1351     // TODO: Unit test that has some Cells in scanner and some in the proto.
1352     List<Cell> cells = null;
1353     if (proto.hasAssociatedCellCount()) {
1354       int count = proto.getAssociatedCellCount();
1355       cells = new ArrayList<Cell>(count + values.size());
1356       for (int i = 0; i < count; i++) {
1357         if (!scanner.advance()) throw new IOException("Failed get " + i + " of " + count);
1358         cells.add(scanner.current());
1359       }
1360     }
1361 
1362     if (!values.isEmpty()){
1363       if (cells == null) cells = new ArrayList<Cell>(values.size());
1364       for (CellProtos.Cell c: values) {
1365         cells.add(toCell(c));
1366       }
1367     }
1368 
1369     return (cells == null || cells.isEmpty())
1370         ? (proto.getStale() ? EMPTY_RESULT_STALE : EMPTY_RESULT)
1371         : Result.create(cells, null, proto.getStale());
1372   }
1373 
1374 
1375   /**
1376    * Convert a ByteArrayComparable to a protocol buffer Comparator
1377    *
1378    * @param comparator the ByteArrayComparable to convert
1379    * @return the converted protocol buffer Comparator
1380    */
1381   public static ComparatorProtos.Comparator toComparator(ByteArrayComparable comparator) {
1382     ComparatorProtos.Comparator.Builder builder = ComparatorProtos.Comparator.newBuilder();
1383     builder.setName(comparator.getClass().getName());
1384     builder.setSerializedComparator(ByteStringer.wrap(comparator.toByteArray()));
1385     return builder.build();
1386   }
1387 
1388   /**
1389    * Convert a protocol buffer Comparator to a ByteArrayComparable
1390    *
1391    * @param proto the protocol buffer Comparator to convert
1392    * @return the converted ByteArrayComparable
1393    */
1394   @SuppressWarnings("unchecked")
1395   public static ByteArrayComparable toComparator(ComparatorProtos.Comparator proto)
1396   throws IOException {
1397     String type = proto.getName();
1398     String funcName = "parseFrom";
1399     byte [] value = proto.getSerializedComparator().toByteArray();
1400     try {
1401       Class<? extends ByteArrayComparable> c =
1402         (Class<? extends ByteArrayComparable>)Class.forName(type, true, CLASS_LOADER);
1403       Method parseFrom = c.getMethod(funcName, byte[].class);
1404       if (parseFrom == null) {
1405         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1406       }
1407       return (ByteArrayComparable)parseFrom.invoke(null, value);
1408     } catch (Exception e) {
1409       throw new IOException(e);
1410     }
1411   }
1412 
1413   /**
1414    * Convert a protocol buffer Filter to a client Filter
1415    *
1416    * @param proto the protocol buffer Filter to convert
1417    * @return the converted Filter
1418    */
1419   @SuppressWarnings("unchecked")
1420   public static Filter toFilter(FilterProtos.Filter proto) throws IOException {
1421     String type = proto.getName();
1422     final byte [] value = proto.getSerializedFilter().toByteArray();
1423     String funcName = "parseFrom";
1424     try {
1425       Class<? extends Filter> c =
1426         (Class<? extends Filter>)Class.forName(type, true, CLASS_LOADER);
1427       Method parseFrom = c.getMethod(funcName, byte[].class);
1428       if (parseFrom == null) {
1429         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1430       }
1431       return (Filter)parseFrom.invoke(c, value);
1432     } catch (Exception e) {
1433       throw new IOException(e);
1434     }
1435   }
1436 
1437   /**
1438    * Convert a client Filter to a protocol buffer Filter
1439    *
1440    * @param filter the Filter to convert
1441    * @return the converted protocol buffer Filter
1442    */
1443   public static FilterProtos.Filter toFilter(Filter filter) throws IOException {
1444     FilterProtos.Filter.Builder builder = FilterProtos.Filter.newBuilder();
1445     builder.setName(filter.getClass().getName());
1446     builder.setSerializedFilter(ByteStringer.wrap(filter.toByteArray()));
1447     return builder.build();
1448   }
1449 
1450   /**
1451    * Convert a delete KeyValue type to protocol buffer DeleteType.
1452    *
1453    * @param type
1454    * @return protocol buffer DeleteType
1455    * @throws IOException
1456    */
1457   public static DeleteType toDeleteType(
1458       KeyValue.Type type) throws IOException {
1459     switch (type) {
1460     case Delete:
1461       return DeleteType.DELETE_ONE_VERSION;
1462     case DeleteColumn:
1463       return DeleteType.DELETE_MULTIPLE_VERSIONS;
1464     case DeleteFamily:
1465       return DeleteType.DELETE_FAMILY;
1466     case DeleteFamilyVersion:
1467       return DeleteType.DELETE_FAMILY_VERSION;
1468     default:
1469         throw new IOException("Unknown delete type: " + type);
1470     }
1471   }
1472 
1473   /**
1474    * Convert a stringified protocol buffer exception Parameter to a Java Exception
1475    *
1476    * @param parameter the protocol buffer Parameter to convert
1477    * @return the converted Exception
1478    * @throws IOException if failed to deserialize the parameter
1479    */
1480   @SuppressWarnings("unchecked")
1481   public static Throwable toException(final NameBytesPair parameter) throws IOException {
1482     if (parameter == null || !parameter.hasValue()) return null;
1483     String desc = parameter.getValue().toStringUtf8();
1484     String type = parameter.getName();
1485     try {
1486       Class<? extends Throwable> c =
1487         (Class<? extends Throwable>)Class.forName(type, true, CLASS_LOADER);
1488       Constructor<? extends Throwable> cn = null;
1489       try {
1490         cn = c.getDeclaredConstructor(String.class);
1491         return cn.newInstance(desc);
1492       } catch (NoSuchMethodException e) {
1493         // Could be a raw RemoteException. See HBASE-8987.
1494         cn = c.getDeclaredConstructor(String.class, String.class);
1495         return cn.newInstance(type, desc);
1496       }
1497     } catch (Exception e) {
1498       throw new IOException(e);
1499     }
1500   }
1501 
1502 // Start helpers for Client
1503 
1504   /**
1505    * A helper to get a row of the closet one before using client protocol.
1506    *
1507    * @param client
1508    * @param regionName
1509    * @param row
1510    * @param family
1511    * @return the row or the closestRowBefore if it doesn't exist
1512    * @throws IOException
1513    * @deprecated since 0.99 - use reversed scanner instead.
1514    */
1515   @Deprecated
1516   public static Result getRowOrBefore(final ClientService.BlockingInterface client,
1517       final byte[] regionName, final byte[] row,
1518       final byte[] family) throws IOException {
1519     GetRequest request =
1520       RequestConverter.buildGetRowOrBeforeRequest(
1521         regionName, row, family);
1522     try {
1523       GetResponse response = client.get(null, request);
1524       if (!response.hasResult()) return null;
1525       return toResult(response.getResult());
1526     } catch (ServiceException se) {
1527       throw getRemoteException(se);
1528     }
1529   }
1530 
1531   /**
1532    * A helper to bulk load a list of HFiles using client protocol.
1533    *
1534    * @param client
1535    * @param familyPaths
1536    * @param regionName
1537    * @param assignSeqNum
1538    * @return true if all are loaded
1539    * @throws IOException
1540    */
1541   public static boolean bulkLoadHFile(final ClientService.BlockingInterface client,
1542       final List<Pair<byte[], String>> familyPaths,
1543       final byte[] regionName, boolean assignSeqNum) throws IOException {
1544     BulkLoadHFileRequest request =
1545       RequestConverter.buildBulkLoadHFileRequest(familyPaths, regionName, assignSeqNum);
1546     try {
1547       BulkLoadHFileResponse response =
1548         client.bulkLoadHFile(null, request);
1549       return response.getLoaded();
1550     } catch (ServiceException se) {
1551       throw getRemoteException(se);
1552     }
1553   }
1554 
1555   public static CoprocessorServiceResponse execService(final ClientService.BlockingInterface client,
1556       final CoprocessorServiceCall call, final byte[] regionName) throws IOException {
1557     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1558         .setCall(call).setRegion(
1559             RequestConverter.buildRegionSpecifier(REGION_NAME, regionName)).build();
1560     try {
1561       CoprocessorServiceResponse response =
1562           client.execService(null, request);
1563       return response;
1564     } catch (ServiceException se) {
1565       throw getRemoteException(se);
1566     }
1567   }
1568 
1569   public static CoprocessorServiceResponse execService(
1570     final MasterService.BlockingInterface client, final CoprocessorServiceCall call)
1571   throws IOException {
1572     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1573         .setCall(call).setRegion(
1574             RequestConverter.buildRegionSpecifier(REGION_NAME, HConstants.EMPTY_BYTE_ARRAY)).build();
1575     try {
1576       CoprocessorServiceResponse response =
1577           client.execMasterService(null, request);
1578       return response;
1579     } catch (ServiceException se) {
1580       throw getRemoteException(se);
1581     }
1582   }
1583 
1584   @SuppressWarnings("unchecked")
1585   public static <T extends Service> T newServiceStub(Class<T> service, RpcChannel channel)
1586       throws Exception {
1587     return (T)Methods.call(service, null, "newStub",
1588         new Class[]{ RpcChannel.class }, new Object[]{ channel });
1589   }
1590 
1591 // End helpers for Client
1592 // Start helpers for Admin
1593 
1594   /**
1595    * A helper to retrieve region info given a region name
1596    * using admin protocol.
1597    *
1598    * @param admin
1599    * @param regionName
1600    * @return the retrieved region info
1601    * @throws IOException
1602    */
1603   public static HRegionInfo getRegionInfo(final AdminService.BlockingInterface admin,
1604       final byte[] regionName) throws IOException {
1605     try {
1606       GetRegionInfoRequest request =
1607         RequestConverter.buildGetRegionInfoRequest(regionName);
1608       GetRegionInfoResponse response =
1609         admin.getRegionInfo(null, request);
1610       return HRegionInfo.convert(response.getRegionInfo());
1611     } catch (ServiceException se) {
1612       throw getRemoteException(se);
1613     }
1614   }
1615 
1616   /**
1617    * A helper to close a region given a region name
1618    * using admin protocol.
1619    *
1620    * @param admin
1621    * @param regionName
1622    * @throws IOException
1623    */
1624   public static void closeRegion(final AdminService.BlockingInterface admin,
1625       final ServerName server, final byte[] regionName) throws IOException {
1626     CloseRegionRequest closeRegionRequest =
1627       RequestConverter.buildCloseRegionRequest(server, regionName);
1628     try {
1629       admin.closeRegion(null, closeRegionRequest);
1630     } catch (ServiceException se) {
1631       throw getRemoteException(se);
1632     }
1633   }
1634 
1635   /**
1636    * A helper to close a region given a region name
1637    * using admin protocol.
1638    *
1639    * @param admin
1640    * @param regionName
1641    * @return true if the region is closed
1642    * @throws IOException
1643    */
1644   public static boolean closeRegion(final AdminService.BlockingInterface admin,
1645       final ServerName server, final byte[] regionName,
1646       final ServerName destinationServer) throws IOException {
1647     CloseRegionRequest closeRegionRequest =
1648       RequestConverter.buildCloseRegionRequest(server,
1649         regionName, destinationServer);
1650     try {
1651       CloseRegionResponse response = admin.closeRegion(null, closeRegionRequest);
1652       return ResponseConverter.isClosed(response);
1653     } catch (ServiceException se) {
1654       throw getRemoteException(se);
1655     }
1656   }
1657 
1658 
1659   /**
1660    * A helper to open a region using admin protocol.
1661    * @param admin
1662    * @param region
1663    * @throws IOException
1664    */
1665   public static void openRegion(final AdminService.BlockingInterface admin,
1666       ServerName server, final HRegionInfo region) throws IOException {
1667     OpenRegionRequest request =
1668       RequestConverter.buildOpenRegionRequest(server, region, null, null);
1669     try {
1670       admin.openRegion(null, request);
1671     } catch (ServiceException se) {
1672       throw ProtobufUtil.getRemoteException(se);
1673     }
1674   }
1675 
1676   /**
1677    * A helper to get the all the online regions on a region
1678    * server using admin protocol.
1679    *
1680    * @param admin
1681    * @return a list of online region info
1682    * @throws IOException
1683    */
1684   public static List<HRegionInfo> getOnlineRegions(final AdminService.BlockingInterface admin)
1685   throws IOException {
1686     GetOnlineRegionRequest request = RequestConverter.buildGetOnlineRegionRequest();
1687     GetOnlineRegionResponse response = null;
1688     try {
1689       response = admin.getOnlineRegion(null, request);
1690     } catch (ServiceException se) {
1691       throw getRemoteException(se);
1692     }
1693     return getRegionInfos(response);
1694   }
1695 
1696   /**
1697    * Get the list of region info from a GetOnlineRegionResponse
1698    *
1699    * @param proto the GetOnlineRegionResponse
1700    * @return the list of region info or null if <code>proto</code> is null
1701    */
1702   static List<HRegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
1703     if (proto == null) return null;
1704     List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
1705     for (RegionInfo regionInfo: proto.getRegionInfoList()) {
1706       regionInfos.add(HRegionInfo.convert(regionInfo));
1707     }
1708     return regionInfos;
1709   }
1710 
1711   /**
1712    * A helper to get the info of a region server using admin protocol.
1713    *
1714    * @param admin
1715    * @return the server name
1716    * @throws IOException
1717    */
1718   public static ServerInfo getServerInfo(final AdminService.BlockingInterface admin)
1719   throws IOException {
1720     GetServerInfoRequest request = RequestConverter.buildGetServerInfoRequest();
1721     try {
1722       GetServerInfoResponse response = admin.getServerInfo(null, request);
1723       return response.getServerInfo();
1724     } catch (ServiceException se) {
1725       throw getRemoteException(se);
1726     }
1727   }
1728 
1729   /**
1730    * A helper to get the list of files of a column family
1731    * on a given region using admin protocol.
1732    *
1733    * @param admin
1734    * @param regionName
1735    * @param family
1736    * @return the list of store files
1737    * @throws IOException
1738    */
1739   public static List<String> getStoreFiles(final AdminService.BlockingInterface admin,
1740       final byte[] regionName, final byte[] family)
1741   throws IOException {
1742     GetStoreFileRequest request =
1743       RequestConverter.buildGetStoreFileRequest(regionName, family);
1744     try {
1745       GetStoreFileResponse response = admin.getStoreFile(null, request);
1746       return response.getStoreFileList();
1747     } catch (ServiceException se) {
1748       throw ProtobufUtil.getRemoteException(se);
1749     }
1750   }
1751 
1752   /**
1753    * A helper to split a region using admin protocol.
1754    *
1755    * @param admin
1756    * @param hri
1757    * @param splitPoint
1758    * @throws IOException
1759    */
1760   public static void split(final AdminService.BlockingInterface admin,
1761       final HRegionInfo hri, byte[] splitPoint) throws IOException {
1762     SplitRegionRequest request =
1763       RequestConverter.buildSplitRegionRequest(hri.getRegionName(), splitPoint);
1764     try {
1765       admin.splitRegion(null, request);
1766     } catch (ServiceException se) {
1767       throw ProtobufUtil.getRemoteException(se);
1768     }
1769   }
1770 
1771   /**
1772    * A helper to merge regions using admin protocol. Send request to
1773    * regionserver.
1774    * @param admin
1775    * @param region_a
1776    * @param region_b
1777    * @param forcible true if do a compulsory merge, otherwise we will only merge
1778    *          two adjacent regions
1779    * @throws IOException
1780    */
1781   public static void mergeRegions(final AdminService.BlockingInterface admin,
1782       final HRegionInfo region_a, final HRegionInfo region_b,
1783       final boolean forcible) throws IOException {
1784     MergeRegionsRequest request = RequestConverter.buildMergeRegionsRequest(
1785         region_a.getRegionName(), region_b.getRegionName(),forcible);
1786     try {
1787       admin.mergeRegions(null, request);
1788     } catch (ServiceException se) {
1789       throw ProtobufUtil.getRemoteException(se);
1790     }
1791   }
1792 
1793 // End helpers for Admin
1794 
1795   /*
1796    * Get the total (read + write) requests from a RegionLoad pb
1797    * @param rl - RegionLoad pb
1798    * @return total (read + write) requests
1799    */
1800   public static long getTotalRequestsCount(RegionLoad rl) {
1801     if (rl == null) {
1802       return 0;
1803     }
1804 
1805     return rl.getReadRequestsCount() + rl.getWriteRequestsCount();
1806   }
1807 
1808 
1809   /**
1810    * @param m Message to get delimited pb serialization of (with pb magic prefix)
1811    */
1812   public static byte [] toDelimitedByteArray(final Message m) throws IOException {
1813     // Allocate arbitrary big size so we avoid resizing.
1814     ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
1815     baos.write(PB_MAGIC);
1816     m.writeDelimitedTo(baos);
1817     return baos.toByteArray();
1818   }
1819 
1820   /**
1821    * Converts a Permission proto to a client Permission object.
1822    *
1823    * @param proto the protobuf Permission
1824    * @return the converted Permission
1825    */
1826   public static Permission toPermission(AccessControlProtos.Permission proto) {
1827     if (proto.getType() != AccessControlProtos.Permission.Type.Global) {
1828       return toTablePermission(proto);
1829     } else {
1830       List<Permission.Action> actions = toPermissionActions(proto.getGlobalPermission().getActionList());
1831       return new Permission(actions.toArray(new Permission.Action[actions.size()]));
1832     }
1833   }
1834 
1835   /**
1836    * Converts a Permission proto to a client TablePermission object.
1837    *
1838    * @param proto the protobuf Permission
1839    * @return the converted TablePermission
1840    */
1841   public static TablePermission toTablePermission(AccessControlProtos.Permission proto) {
1842     if(proto.getType() == AccessControlProtos.Permission.Type.Global) {
1843       AccessControlProtos.GlobalPermission perm = proto.getGlobalPermission();
1844       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1845 
1846       return new TablePermission(null, null, null,
1847           actions.toArray(new Permission.Action[actions.size()]));
1848     }
1849     if(proto.getType() == AccessControlProtos.Permission.Type.Namespace) {
1850       AccessControlProtos.NamespacePermission perm = proto.getNamespacePermission();
1851       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1852 
1853       if(!proto.hasNamespacePermission()) {
1854         throw new IllegalStateException("Namespace must not be empty in NamespacePermission");
1855       }
1856       String namespace = perm.getNamespaceName().toStringUtf8();
1857       return new TablePermission(namespace, actions.toArray(new Permission.Action[actions.size()]));
1858     }
1859     if(proto.getType() == AccessControlProtos.Permission.Type.Table) {
1860       AccessControlProtos.TablePermission perm = proto.getTablePermission();
1861       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1862 
1863       byte[] qualifier = null;
1864       byte[] family = null;
1865       TableName table = null;
1866 
1867       if (!perm.hasTableName()) {
1868         throw new IllegalStateException("TableName cannot be empty");
1869       }
1870       table = ProtobufUtil.toTableName(perm.getTableName());
1871 
1872       if (perm.hasFamily()) family = perm.getFamily().toByteArray();
1873       if (perm.hasQualifier()) qualifier = perm.getQualifier().toByteArray();
1874 
1875       return new TablePermission(table, family, qualifier,
1876           actions.toArray(new Permission.Action[actions.size()]));
1877     }
1878     throw new IllegalStateException("Unrecognize Perm Type: "+proto.getType());
1879   }
1880 
1881   /**
1882    * Convert a client Permission to a Permission proto
1883    *
1884    * @param perm the client Permission
1885    * @return the protobuf Permission
1886    */
1887   public static AccessControlProtos.Permission toPermission(Permission perm) {
1888     AccessControlProtos.Permission.Builder ret = AccessControlProtos.Permission.newBuilder();
1889     if (perm instanceof TablePermission) {
1890       TablePermission tablePerm = (TablePermission)perm;
1891       if(tablePerm.hasNamespace()) {
1892         ret.setType(AccessControlProtos.Permission.Type.Namespace);
1893 
1894         AccessControlProtos.NamespacePermission.Builder builder =
1895             AccessControlProtos.NamespacePermission.newBuilder();
1896         builder.setNamespaceName(ByteString.copyFromUtf8(tablePerm.getNamespace()));
1897         Permission.Action actions[] = perm.getActions();
1898         if (actions != null) {
1899           for (Permission.Action a : actions) {
1900             builder.addAction(toPermissionAction(a));
1901           }
1902 	}
1903         ret.setNamespacePermission(builder);
1904         return ret.build();
1905       } else if (tablePerm.hasTable()) {
1906         ret.setType(AccessControlProtos.Permission.Type.Table);
1907 
1908         AccessControlProtos.TablePermission.Builder builder =
1909             AccessControlProtos.TablePermission.newBuilder();
1910         builder.setTableName(ProtobufUtil.toProtoTableName(tablePerm.getTableName()));
1911         if (tablePerm.hasFamily()) {
1912           builder.setFamily(ByteStringer.wrap(tablePerm.getFamily()));
1913         }
1914         if (tablePerm.hasQualifier()) {
1915           builder.setQualifier(ByteStringer.wrap(tablePerm.getQualifier()));
1916         }
1917         Permission.Action actions[] = perm.getActions();
1918         if (actions != null) {
1919           for (Permission.Action a : actions) {
1920             builder.addAction(toPermissionAction(a));
1921           }
1922         }
1923         ret.setTablePermission(builder);
1924         return ret.build();
1925       }
1926     }
1927 
1928     ret.setType(AccessControlProtos.Permission.Type.Global);
1929 
1930     AccessControlProtos.GlobalPermission.Builder builder =
1931         AccessControlProtos.GlobalPermission.newBuilder();
1932     Permission.Action actions[] = perm.getActions();
1933     if (actions != null) {
1934       for (Permission.Action a: actions) {
1935         builder.addAction(toPermissionAction(a));
1936       }
1937     }
1938     ret.setGlobalPermission(builder);
1939     return ret.build();
1940   }
1941 
1942   /**
1943    * Converts a list of Permission.Action proto to a list of client Permission.Action objects.
1944    *
1945    * @param protoActions the list of protobuf Actions
1946    * @return the converted list of Actions
1947    */
1948   public static List<Permission.Action> toPermissionActions(
1949       List<AccessControlProtos.Permission.Action> protoActions) {
1950     List<Permission.Action> actions = new ArrayList<Permission.Action>(protoActions.size());
1951     for (AccessControlProtos.Permission.Action a : protoActions) {
1952       actions.add(toPermissionAction(a));
1953     }
1954     return actions;
1955   }
1956 
1957   /**
1958    * Converts a Permission.Action proto to a client Permission.Action object.
1959    *
1960    * @param action the protobuf Action
1961    * @return the converted Action
1962    */
1963   public static Permission.Action toPermissionAction(
1964       AccessControlProtos.Permission.Action action) {
1965     switch (action) {
1966       case READ:
1967         return Permission.Action.READ;
1968       case WRITE:
1969         return Permission.Action.WRITE;
1970       case EXEC:
1971         return Permission.Action.EXEC;
1972       case CREATE:
1973         return Permission.Action.CREATE;
1974       case ADMIN:
1975         return Permission.Action.ADMIN;
1976     }
1977     throw new IllegalArgumentException("Unknown action value "+action.name());
1978   }
1979 
1980   /**
1981    * Convert a client Permission.Action to a Permission.Action proto
1982    *
1983    * @param action the client Action
1984    * @return the protobuf Action
1985    */
1986   public static AccessControlProtos.Permission.Action toPermissionAction(
1987       Permission.Action action) {
1988     switch (action) {
1989       case READ:
1990         return AccessControlProtos.Permission.Action.READ;
1991       case WRITE:
1992         return AccessControlProtos.Permission.Action.WRITE;
1993       case EXEC:
1994         return AccessControlProtos.Permission.Action.EXEC;
1995       case CREATE:
1996         return AccessControlProtos.Permission.Action.CREATE;
1997       case ADMIN:
1998         return AccessControlProtos.Permission.Action.ADMIN;
1999     }
2000     throw new IllegalArgumentException("Unknown action value "+action.name());
2001   }
2002 
2003   /**
2004    * Convert a client user permission to a user permission proto
2005    *
2006    * @param perm the client UserPermission
2007    * @return the protobuf UserPermission
2008    */
2009   public static AccessControlProtos.UserPermission toUserPermission(UserPermission perm) {
2010     return AccessControlProtos.UserPermission.newBuilder()
2011         .setUser(ByteStringer.wrap(perm.getUser()))
2012         .setPermission(toPermission(perm))
2013         .build();
2014   }
2015 
2016   /**
2017    * Converts a user permission proto to a client user permission object.
2018    *
2019    * @param proto the protobuf UserPermission
2020    * @return the converted UserPermission
2021    */
2022   public static UserPermission toUserPermission(AccessControlProtos.UserPermission proto) {
2023     return new UserPermission(proto.getUser().toByteArray(),
2024         toTablePermission(proto.getPermission()));
2025   }
2026 
2027   /**
2028    * Convert a ListMultimap<String, TablePermission> where key is username
2029    * to a protobuf UserPermission
2030    *
2031    * @param perm the list of user and table permissions
2032    * @return the protobuf UserTablePermissions
2033    */
2034   public static AccessControlProtos.UsersAndPermissions toUserTablePermissions(
2035       ListMultimap<String, TablePermission> perm) {
2036     AccessControlProtos.UsersAndPermissions.Builder builder =
2037                   AccessControlProtos.UsersAndPermissions.newBuilder();
2038     for (Map.Entry<String, Collection<TablePermission>> entry : perm.asMap().entrySet()) {
2039       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
2040                   AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
2041       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
2042       for (TablePermission tablePerm: entry.getValue()) {
2043         userPermBuilder.addPermissions(toPermission(tablePerm));
2044       }
2045       builder.addUserPermissions(userPermBuilder.build());
2046     }
2047     return builder.build();
2048   }
2049 
2050   /**
2051    * A utility used to grant a user global permissions.
2052    * <p>
2053    * It's also called by the shell, in case you want to find references.
2054    *
2055    * @param protocol the AccessControlService protocol proxy
2056    * @param userShortName the short name of the user to grant permissions
2057    * @param actions the permissions to be granted
2058    * @throws ServiceException
2059    */
2060   public static void grant(AccessControlService.BlockingInterface protocol,
2061       String userShortName, Permission.Action... actions) throws ServiceException {
2062     List<AccessControlProtos.Permission.Action> permActions =
2063         Lists.newArrayListWithCapacity(actions.length);
2064     for (Permission.Action a : actions) {
2065       permActions.add(ProtobufUtil.toPermissionAction(a));
2066     }
2067     AccessControlProtos.GrantRequest request = RequestConverter.
2068       buildGrantRequest(userShortName, permActions.toArray(
2069         new AccessControlProtos.Permission.Action[actions.length]));
2070     protocol.grant(null, request);
2071   }
2072 
2073   /**
2074    * A utility used to grant a user table permissions. The permissions will
2075    * be for a table table/column family/qualifier.
2076    * <p>
2077    * It's also called by the shell, in case you want to find references.
2078    *
2079    * @param protocol the AccessControlService protocol proxy
2080    * @param userShortName the short name of the user to grant permissions
2081    * @param tableName optional table name
2082    * @param f optional column family
2083    * @param q optional qualifier
2084    * @param actions the permissions to be granted
2085    * @throws ServiceException
2086    */
2087   public static void grant(AccessControlService.BlockingInterface protocol,
2088       String userShortName, TableName tableName, byte[] f, byte[] q,
2089       Permission.Action... actions) throws ServiceException {
2090     List<AccessControlProtos.Permission.Action> permActions =
2091         Lists.newArrayListWithCapacity(actions.length);
2092     for (Permission.Action a : actions) {
2093       permActions.add(ProtobufUtil.toPermissionAction(a));
2094     }
2095     AccessControlProtos.GrantRequest request = RequestConverter.
2096       buildGrantRequest(userShortName, tableName, f, q, permActions.toArray(
2097         new AccessControlProtos.Permission.Action[actions.length]));
2098     protocol.grant(null, request);
2099   }
2100 
2101   /**
2102    * A utility used to grant a user namespace permissions.
2103    * <p>
2104    * It's also called by the shell, in case you want to find references.
2105    *
2106    * @param protocol the AccessControlService protocol proxy
2107    * @param namespace the short name of the user to grant permissions
2108    * @param actions the permissions to be granted
2109    * @throws ServiceException
2110    */
2111   public static void grant(AccessControlService.BlockingInterface protocol,
2112       String userShortName, String namespace,
2113       Permission.Action... actions) throws ServiceException {
2114     List<AccessControlProtos.Permission.Action> permActions =
2115         Lists.newArrayListWithCapacity(actions.length);
2116     for (Permission.Action a : actions) {
2117       permActions.add(ProtobufUtil.toPermissionAction(a));
2118     }
2119     AccessControlProtos.GrantRequest request = RequestConverter.
2120       buildGrantRequest(userShortName, namespace, permActions.toArray(
2121         new AccessControlProtos.Permission.Action[actions.length]));
2122     protocol.grant(null, request);
2123   }
2124 
2125   /**
2126    * A utility used to revoke a user's global permissions.
2127    * <p>
2128    * It's also called by the shell, in case you want to find references.
2129    *
2130    * @param protocol the AccessControlService protocol proxy
2131    * @param userShortName the short name of the user to revoke permissions
2132    * @param actions the permissions to be revoked
2133    * @throws ServiceException
2134    */
2135   public static void revoke(AccessControlService.BlockingInterface protocol,
2136       String userShortName, Permission.Action... actions) throws ServiceException {
2137     List<AccessControlProtos.Permission.Action> permActions =
2138         Lists.newArrayListWithCapacity(actions.length);
2139     for (Permission.Action a : actions) {
2140       permActions.add(ProtobufUtil.toPermissionAction(a));
2141     }
2142     AccessControlProtos.RevokeRequest request = RequestConverter.
2143       buildRevokeRequest(userShortName, permActions.toArray(
2144         new AccessControlProtos.Permission.Action[actions.length]));
2145     protocol.revoke(null, request);
2146   }
2147 
2148   /**
2149    * A utility used to revoke a user's table permissions. The permissions will
2150    * be for a table/column family/qualifier.
2151    * <p>
2152    * It's also called by the shell, in case you want to find references.
2153    *
2154    * @param protocol the AccessControlService protocol proxy
2155    * @param userShortName the short name of the user to revoke permissions
2156    * @param tableName optional table name
2157    * @param f optional column family
2158    * @param q optional qualifier
2159    * @param actions the permissions to be revoked
2160    * @throws ServiceException
2161    */
2162   public static void revoke(AccessControlService.BlockingInterface protocol,
2163       String userShortName, TableName tableName, byte[] f, byte[] q,
2164       Permission.Action... actions) throws ServiceException {
2165     List<AccessControlProtos.Permission.Action> permActions =
2166         Lists.newArrayListWithCapacity(actions.length);
2167     for (Permission.Action a : actions) {
2168       permActions.add(ProtobufUtil.toPermissionAction(a));
2169     }
2170     AccessControlProtos.RevokeRequest request = RequestConverter.
2171       buildRevokeRequest(userShortName, tableName, f, q, permActions.toArray(
2172         new AccessControlProtos.Permission.Action[actions.length]));
2173     protocol.revoke(null, request);
2174   }
2175 
2176   /**
2177    * A utility used to revoke a user's namespace permissions.
2178    * <p>
2179    * It's also called by the shell, in case you want to find references.
2180    *
2181    * @param protocol the AccessControlService protocol proxy
2182    * @param userShortName the short name of the user to revoke permissions
2183    * @param namespace optional table name
2184    * @param actions the permissions to be revoked
2185    * @throws ServiceException
2186    */
2187   public static void revoke(AccessControlService.BlockingInterface protocol,
2188       String userShortName, String namespace,
2189       Permission.Action... actions) throws ServiceException {
2190     List<AccessControlProtos.Permission.Action> permActions =
2191         Lists.newArrayListWithCapacity(actions.length);
2192     for (Permission.Action a : actions) {
2193       permActions.add(ProtobufUtil.toPermissionAction(a));
2194     }
2195     AccessControlProtos.RevokeRequest request = RequestConverter.
2196       buildRevokeRequest(userShortName, namespace, permActions.toArray(
2197         new AccessControlProtos.Permission.Action[actions.length]));
2198     protocol.revoke(null, request);
2199   }
2200 
2201   /**
2202    * A utility used to get user's global permissions.
2203    * <p>
2204    * It's also called by the shell, in case you want to find references.
2205    *
2206    * @param protocol the AccessControlService protocol proxy
2207    * @throws ServiceException
2208    */
2209   public static List<UserPermission> getUserPermissions(
2210       AccessControlService.BlockingInterface protocol) throws ServiceException {
2211     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2212       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2213     builder.setType(AccessControlProtos.Permission.Type.Global);
2214     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2215     AccessControlProtos.GetUserPermissionsResponse response =
2216       protocol.getUserPermissions(null, request);
2217     List<UserPermission> perms = new ArrayList<UserPermission>();
2218     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2219       perms.add(ProtobufUtil.toUserPermission(perm));
2220     }
2221     return perms;
2222   }
2223 
2224   /**
2225    * A utility used to get user table permissions.
2226    * <p>
2227    * It's also called by the shell, in case you want to find references.
2228    *
2229    * @param protocol the AccessControlService protocol proxy
2230    * @param t optional table name
2231    * @throws ServiceException
2232    */
2233   public static List<UserPermission> getUserPermissions(
2234       AccessControlService.BlockingInterface protocol,
2235       TableName t) throws ServiceException {
2236     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2237       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2238     if (t != null) {
2239       builder.setTableName(ProtobufUtil.toProtoTableName(t));
2240     }
2241     builder.setType(AccessControlProtos.Permission.Type.Table);
2242     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2243     AccessControlProtos.GetUserPermissionsResponse response =
2244       protocol.getUserPermissions(null, request);
2245     List<UserPermission> perms = new ArrayList<UserPermission>();
2246     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2247       perms.add(ProtobufUtil.toUserPermission(perm));
2248     }
2249     return perms;
2250   }
2251 
2252   /**
2253    * A utility used to get permissions for selected namespace.
2254    * <p>
2255    * It's also called by the shell, in case you want to find references.
2256    *
2257    * @param protocol the AccessControlService protocol proxy
2258    * @param namespace name of the namespace
2259    * @throws ServiceException
2260    */
2261   public static List<UserPermission> getUserPermissions(
2262       AccessControlService.BlockingInterface protocol,
2263       byte[] namespace) throws ServiceException {
2264     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2265       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2266     if (namespace != null) {
2267       builder.setNamespaceName(ByteStringer.wrap(namespace));
2268     }
2269     builder.setType(AccessControlProtos.Permission.Type.Namespace);
2270     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2271     AccessControlProtos.GetUserPermissionsResponse response =
2272       protocol.getUserPermissions(null, request);
2273     List<UserPermission> perms = new ArrayList<UserPermission>();
2274     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2275       perms.add(ProtobufUtil.toUserPermission(perm));
2276     }
2277     return perms;
2278   }
2279 
2280   /**
2281    * Convert a protobuf UserTablePermissions to a
2282    * ListMultimap<String, TablePermission> where key is username.
2283    *
2284    * @param proto the protobuf UserPermission
2285    * @return the converted UserPermission
2286    */
2287   public static ListMultimap<String, TablePermission> toUserTablePermissions(
2288       AccessControlProtos.UsersAndPermissions proto) {
2289     ListMultimap<String, TablePermission> perms = ArrayListMultimap.create();
2290     AccessControlProtos.UsersAndPermissions.UserPermissions userPerm;
2291 
2292     for (int i = 0; i < proto.getUserPermissionsCount(); i++) {
2293       userPerm = proto.getUserPermissions(i);
2294       for (int j = 0; j < userPerm.getPermissionsCount(); j++) {
2295         TablePermission tablePerm = toTablePermission(userPerm.getPermissions(j));
2296         perms.put(userPerm.getUser().toStringUtf8(), tablePerm);
2297       }
2298     }
2299 
2300     return perms;
2301   }
2302 
2303   /**
2304    * Converts a Token instance (with embedded identifier) to the protobuf representation.
2305    *
2306    * @param token the Token instance to copy
2307    * @return the protobuf Token message
2308    */
2309   public static AuthenticationProtos.Token toToken(Token<AuthenticationTokenIdentifier> token) {
2310     AuthenticationProtos.Token.Builder builder = AuthenticationProtos.Token.newBuilder();
2311     builder.setIdentifier(ByteStringer.wrap(token.getIdentifier()));
2312     builder.setPassword(ByteStringer.wrap(token.getPassword()));
2313     if (token.getService() != null) {
2314       builder.setService(ByteString.copyFromUtf8(token.getService().toString()));
2315     }
2316     return builder.build();
2317   }
2318 
2319   /**
2320    * Converts a protobuf Token message back into a Token instance.
2321    *
2322    * @param proto the protobuf Token message
2323    * @return the Token instance
2324    */
2325   public static Token<AuthenticationTokenIdentifier> toToken(AuthenticationProtos.Token proto) {
2326     return new Token<AuthenticationTokenIdentifier>(
2327         proto.hasIdentifier() ? proto.getIdentifier().toByteArray() : null,
2328         proto.hasPassword() ? proto.getPassword().toByteArray() : null,
2329         AuthenticationTokenIdentifier.AUTH_TOKEN_TYPE,
2330         proto.hasService() ? new Text(proto.getService().toStringUtf8()) : null);
2331   }
2332 
2333   /**
2334    * Find the HRegion encoded name based on a region specifier
2335    *
2336    * @param regionSpecifier the region specifier
2337    * @return the corresponding region's encoded name
2338    * @throws DoNotRetryIOException if the specifier type is unsupported
2339    */
2340   public static String getRegionEncodedName(
2341       final RegionSpecifier regionSpecifier) throws DoNotRetryIOException {
2342     byte[] value = regionSpecifier.getValue().toByteArray();
2343     RegionSpecifierType type = regionSpecifier.getType();
2344     switch (type) {
2345       case REGION_NAME:
2346         return HRegionInfo.encodeRegionName(value);
2347       case ENCODED_REGION_NAME:
2348         return Bytes.toString(value);
2349       default:
2350         throw new DoNotRetryIOException(
2351           "Unsupported region specifier type: " + type);
2352     }
2353   }
2354 
2355   public static ScanMetrics toScanMetrics(final byte[] bytes) {
2356     Parser<MapReduceProtos.ScanMetrics> parser = MapReduceProtos.ScanMetrics.PARSER;
2357     MapReduceProtos.ScanMetrics pScanMetrics = null;
2358     try {
2359       pScanMetrics = parser.parseFrom(bytes);
2360     } catch (InvalidProtocolBufferException e) {
2361       //Ignored there are just no key values to add.
2362     }
2363     ScanMetrics scanMetrics = new ScanMetrics();
2364     if (pScanMetrics != null) {
2365       for (HBaseProtos.NameInt64Pair pair : pScanMetrics.getMetricsList()) {
2366         if (pair.hasName() && pair.hasValue()) {
2367           scanMetrics.setCounter(pair.getName(), pair.getValue());
2368         }
2369       }
2370     }
2371     return scanMetrics;
2372   }
2373 
2374   public static MapReduceProtos.ScanMetrics toScanMetrics(ScanMetrics scanMetrics) {
2375     MapReduceProtos.ScanMetrics.Builder builder = MapReduceProtos.ScanMetrics.newBuilder();
2376     Map<String, Long> metrics = scanMetrics.getMetricsMap();
2377     for (Entry<String, Long> e : metrics.entrySet()) {
2378       HBaseProtos.NameInt64Pair nameInt64Pair =
2379           HBaseProtos.NameInt64Pair.newBuilder()
2380               .setName(e.getKey())
2381               .setValue(e.getValue())
2382               .build();
2383       builder.addMetrics(nameInt64Pair);
2384     }
2385     return builder.build();
2386   }
2387 
2388   /**
2389    * Unwraps an exception from a protobuf service into the underlying (expected) IOException.
2390    * This method will <strong>always</strong> throw an exception.
2391    * @param se the {@code ServiceException} instance to convert into an {@code IOException}
2392    */
2393   public static void toIOException(ServiceException se) throws IOException {
2394     if (se == null) {
2395       throw new NullPointerException("Null service exception passed!");
2396     }
2397 
2398     Throwable cause = se.getCause();
2399     if (cause != null && cause instanceof IOException) {
2400       throw (IOException)cause;
2401     }
2402     throw new IOException(se);
2403   }
2404 
2405   public static CellProtos.Cell toCell(final Cell kv) {
2406     // Doing this is going to kill us if we do it for all data passed.
2407     // St.Ack 20121205
2408     CellProtos.Cell.Builder kvbuilder = CellProtos.Cell.newBuilder();
2409     kvbuilder.setRow(ByteStringer.wrap(kv.getRowArray(), kv.getRowOffset(),
2410         kv.getRowLength()));
2411     kvbuilder.setFamily(ByteStringer.wrap(kv.getFamilyArray(),
2412         kv.getFamilyOffset(), kv.getFamilyLength()));
2413     kvbuilder.setQualifier(ByteStringer.wrap(kv.getQualifierArray(),
2414         kv.getQualifierOffset(), kv.getQualifierLength()));
2415     kvbuilder.setCellType(CellProtos.CellType.valueOf(kv.getTypeByte()));
2416     kvbuilder.setTimestamp(kv.getTimestamp());
2417     kvbuilder.setValue(ByteStringer.wrap(kv.getValueArray(), kv.getValueOffset(),
2418         kv.getValueLength()));
2419     return kvbuilder.build();
2420   }
2421 
2422   public static Cell toCell(final CellProtos.Cell cell) {
2423     // Doing this is going to kill us if we do it for all data passed.
2424     // St.Ack 20121205
2425     return CellUtil.createCell(cell.getRow().toByteArray(),
2426       cell.getFamily().toByteArray(),
2427       cell.getQualifier().toByteArray(),
2428       cell.getTimestamp(),
2429       (byte)cell.getCellType().getNumber(),
2430       cell.getValue().toByteArray());
2431   }
2432 
2433   public static HBaseProtos.NamespaceDescriptor toProtoNamespaceDescriptor(NamespaceDescriptor ns) {
2434     HBaseProtos.NamespaceDescriptor.Builder b =
2435         HBaseProtos.NamespaceDescriptor.newBuilder()
2436             .setName(ByteString.copyFromUtf8(ns.getName()));
2437     for(Map.Entry<String, String> entry: ns.getConfiguration().entrySet()) {
2438       b.addConfiguration(HBaseProtos.NameStringPair.newBuilder()
2439           .setName(entry.getKey())
2440           .setValue(entry.getValue()));
2441     }
2442     return b.build();
2443   }
2444 
2445   public static NamespaceDescriptor toNamespaceDescriptor(
2446       HBaseProtos.NamespaceDescriptor desc) throws IOException {
2447     NamespaceDescriptor.Builder b =
2448       NamespaceDescriptor.create(desc.getName().toStringUtf8());
2449     for(HBaseProtos.NameStringPair prop : desc.getConfigurationList()) {
2450       b.addConfiguration(prop.getName(), prop.getValue());
2451     }
2452     return b.build();
2453   }
2454 
2455   /**
2456    * Get an instance of the argument type declared in a class's signature. The
2457    * argument type is assumed to be a PB Message subclass, and the instance is
2458    * created using parseFrom method on the passed ByteString.
2459    * @param runtimeClass the runtime type of the class
2460    * @param position the position of the argument in the class declaration
2461    * @param b the ByteString which should be parsed to get the instance created
2462    * @return the instance
2463    * @throws IOException
2464    */
2465   @SuppressWarnings("unchecked")
2466   public static <T extends Message>
2467   T getParsedGenericInstance(Class<?> runtimeClass, int position, ByteString b)
2468       throws IOException {
2469     Type type = runtimeClass.getGenericSuperclass();
2470     Type argType = ((ParameterizedType)type).getActualTypeArguments()[position];
2471     Class<T> classType = (Class<T>)argType;
2472     T inst;
2473     try {
2474       Method m = classType.getMethod("parseFrom", ByteString.class);
2475       inst = (T)m.invoke(null, b);
2476       return inst;
2477     } catch (SecurityException e) {
2478       throw new IOException(e);
2479     } catch (NoSuchMethodException e) {
2480       throw new IOException(e);
2481     } catch (IllegalArgumentException e) {
2482       throw new IOException(e);
2483     } catch (InvocationTargetException e) {
2484       throw new IOException(e);
2485     } catch (IllegalAccessException e) {
2486       throw new IOException(e);
2487     }
2488   }
2489 
2490   @SuppressWarnings("deprecation")
2491   public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family,
2492       List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
2493     // compaction descriptor contains relative paths.
2494     // input / output paths are relative to the store dir
2495     // store dir is relative to region dir
2496     CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder()
2497         .setTableName(ByteStringer.wrap(info.getTableName()))
2498         .setEncodedRegionName(ByteStringer.wrap(info.getEncodedNameAsBytes()))
2499         .setFamilyName(ByteStringer.wrap(family))
2500         .setStoreHomeDir(storeDir.getName()); //make relative
2501     for (Path inputPath : inputPaths) {
2502       builder.addCompactionInput(inputPath.getName()); //relative path
2503     }
2504     for (Path outputPath : outputPaths) {
2505       builder.addCompactionOutput(outputPath.getName());
2506     }
2507     builder.setRegionName(ByteStringer.wrap(info.getRegionName()));
2508     return builder.build();
2509   }
2510 
2511   public static FlushDescriptor toFlushDescriptor(FlushAction action, HRegionInfo hri,
2512       long flushSeqId, Map<byte[], List<Path>> committedFiles) {
2513     FlushDescriptor.Builder desc = FlushDescriptor.newBuilder()
2514         .setAction(action)
2515         .setEncodedRegionName(ByteStringer.wrap(hri.getEncodedNameAsBytes()))
2516         .setFlushSequenceNumber(flushSeqId)
2517         .setTableName(ByteStringer.wrap(hri.getTable().getName()));
2518 
2519     for (Map.Entry<byte[], List<Path>> entry : committedFiles.entrySet()) {
2520       WALProtos.FlushDescriptor.StoreFlushDescriptor.Builder builder =
2521           WALProtos.FlushDescriptor.StoreFlushDescriptor.newBuilder()
2522           .setFamilyName(ByteStringer.wrap(entry.getKey()))
2523           .setStoreHomeDir(Bytes.toString(entry.getKey())); //relative to region
2524       if (entry.getValue() != null) {
2525         for (Path path : entry.getValue()) {
2526           builder.addFlushOutput(path.getName());
2527         }
2528       }
2529       desc.addStoreFlushes(builder);
2530     }
2531     return desc.build();
2532   }
2533 
2534   public static RegionEventDescriptor toRegionEventDescriptor(
2535       EventType eventType, HRegionInfo hri, long seqId, ServerName server,
2536       Map<byte[], List<Path>> storeFiles) {
2537     RegionEventDescriptor.Builder desc = RegionEventDescriptor.newBuilder()
2538         .setEventType(eventType)
2539         .setTableName(ByteStringer.wrap(hri.getTable().getName()))
2540         .setEncodedRegionName(ByteStringer.wrap(hri.getEncodedNameAsBytes()))
2541         .setLogSequenceNumber(seqId)
2542         .setServer(toServerName(server));
2543 
2544     for (Map.Entry<byte[], List<Path>> entry : storeFiles.entrySet()) {
2545       RegionEventDescriptor.StoreDescriptor.Builder builder
2546         = RegionEventDescriptor.StoreDescriptor.newBuilder()
2547           .setFamilyName(ByteStringer.wrap(entry.getKey()))
2548           .setStoreHomeDir(Bytes.toString(entry.getKey()));
2549       for (Path path : entry.getValue()) {
2550         builder.addStoreFile(path.getName());
2551       }
2552 
2553       desc.addStores(builder);
2554     }
2555     return desc.build();
2556   }
2557 
2558   /**
2559    * Return short version of Message toString'd, shorter than TextFormat#shortDebugString.
2560    * Tries to NOT print out data both because it can be big but also so we do not have data in our
2561    * logs. Use judiciously.
2562    * @param m
2563    * @return toString of passed <code>m</code>
2564    */
2565   public static String getShortTextFormat(Message m) {
2566     if (m == null) return "null";
2567     if (m instanceof ScanRequest) {
2568       // This should be small and safe to output.  No data.
2569       return TextFormat.shortDebugString(m);
2570     } else if (m instanceof RegionServerReportRequest) {
2571       // Print a short message only, just the servername and the requests, not the full load.
2572       RegionServerReportRequest r = (RegionServerReportRequest)m;
2573       return "server " + TextFormat.shortDebugString(r.getServer()) +
2574         " load { numberOfRequests: " + r.getLoad().getNumberOfRequests() + " }";
2575     } else if (m instanceof RegionServerStartupRequest) {
2576       // Should be small enough.
2577       return TextFormat.shortDebugString(m);
2578     } else if (m instanceof MutationProto) {
2579       return toShortString((MutationProto)m);
2580     } else if (m instanceof GetRequest) {
2581       GetRequest r = (GetRequest) m;
2582       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2583           ", row=" + getStringForByteString(r.getGet().getRow());
2584     } else if (m instanceof ClientProtos.MultiRequest) {
2585       ClientProtos.MultiRequest r = (ClientProtos.MultiRequest) m;
2586       // Get first set of Actions.
2587       ClientProtos.RegionAction actions = r.getRegionActionList().get(0);
2588       String row = actions.getActionCount() <= 0? "":
2589         getStringForByteString(actions.getAction(0).hasGet()?
2590           actions.getAction(0).getGet().getRow():
2591           actions.getAction(0).getMutation().getRow());
2592       return "region= " + getStringForByteString(actions.getRegion().getValue()) +
2593           ", for " + r.getRegionActionCount() +
2594           " actions and 1st row key=" + row;
2595     } else if (m instanceof ClientProtos.MutateRequest) {
2596       ClientProtos.MutateRequest r = (ClientProtos.MutateRequest) m;
2597       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2598           ", row=" + getStringForByteString(r.getMutation().getRow());
2599     }
2600     return "TODO: " + m.getClass().toString();
2601   }
2602 
2603   private static String getStringForByteString(ByteString bs) {
2604     return Bytes.toStringBinary(bs.toByteArray());
2605   }
2606 
2607   /**
2608    * Print out some subset of a MutationProto rather than all of it and its data
2609    * @param proto Protobuf to print out
2610    * @return Short String of mutation proto
2611    */
2612   static String toShortString(final MutationProto proto) {
2613     return "row=" + Bytes.toString(proto.getRow().toByteArray()) +
2614         ", type=" + proto.getMutateType().toString();
2615   }
2616 
2617   public static TableName toTableName(HBaseProtos.TableName tableNamePB) {
2618     return TableName.valueOf(tableNamePB.getNamespace().asReadOnlyByteBuffer(),
2619         tableNamePB.getQualifier().asReadOnlyByteBuffer());
2620   }
2621 
2622   public static HBaseProtos.TableName toProtoTableName(TableName tableName) {
2623     return HBaseProtos.TableName.newBuilder()
2624         .setNamespace(ByteStringer.wrap(tableName.getNamespace()))
2625         .setQualifier(ByteStringer.wrap(tableName.getQualifier())).build();
2626   }
2627 
2628   public static TableName[] getTableNameArray(List<HBaseProtos.TableName> tableNamesList) {
2629     if (tableNamesList == null) {
2630       return new TableName[0];
2631     }
2632     TableName[] tableNames = new TableName[tableNamesList.size()];
2633     for (int i = 0; i < tableNamesList.size(); i++) {
2634       tableNames[i] = toTableName(tableNamesList.get(i));
2635     }
2636     return tableNames;
2637   }
2638 
2639   /**
2640    * Convert a protocol buffer CellVisibility to a client CellVisibility
2641    *
2642    * @param proto
2643    * @return the converted client CellVisibility
2644    */
2645   public static CellVisibility toCellVisibility(ClientProtos.CellVisibility proto) {
2646     if (proto == null) return null;
2647     return new CellVisibility(proto.getExpression());
2648   }
2649 
2650   /**
2651    * Convert a protocol buffer CellVisibility bytes to a client CellVisibility
2652    *
2653    * @param protoBytes
2654    * @return the converted client CellVisibility
2655    * @throws DeserializationException
2656    */
2657   public static CellVisibility toCellVisibility(byte[] protoBytes) throws DeserializationException {
2658     if (protoBytes == null) return null;
2659     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2660     ClientProtos.CellVisibility proto = null;
2661     try {
2662       proto = builder.mergeFrom(protoBytes).build();
2663     } catch (InvalidProtocolBufferException e) {
2664       throw new DeserializationException(e);
2665     }
2666     return toCellVisibility(proto);
2667   }
2668 
2669   /**
2670    * Create a protocol buffer CellVisibility based on a client CellVisibility.
2671    *
2672    * @param cellVisibility
2673    * @return a protocol buffer CellVisibility
2674    */
2675   public static ClientProtos.CellVisibility toCellVisibility(CellVisibility cellVisibility) {
2676     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2677     builder.setExpression(cellVisibility.getExpression());
2678     return builder.build();
2679   }
2680 
2681   /**
2682    * Convert a protocol buffer Authorizations to a client Authorizations
2683    *
2684    * @param proto
2685    * @return the converted client Authorizations
2686    */
2687   public static Authorizations toAuthorizations(ClientProtos.Authorizations proto) {
2688     if (proto == null) return null;
2689     return new Authorizations(proto.getLabelList());
2690   }
2691 
2692   /**
2693    * Convert a protocol buffer Authorizations bytes to a client Authorizations
2694    *
2695    * @param protoBytes
2696    * @return the converted client Authorizations
2697    * @throws DeserializationException
2698    */
2699   public static Authorizations toAuthorizations(byte[] protoBytes) throws DeserializationException {
2700     if (protoBytes == null) return null;
2701     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2702     ClientProtos.Authorizations proto = null;
2703     try {
2704       proto = builder.mergeFrom(protoBytes).build();
2705     } catch (InvalidProtocolBufferException e) {
2706       throw new DeserializationException(e);
2707     }
2708     return toAuthorizations(proto);
2709   }
2710 
2711   /**
2712    * Create a protocol buffer Authorizations based on a client Authorizations.
2713    *
2714    * @param authorizations
2715    * @return a protocol buffer Authorizations
2716    */
2717   public static ClientProtos.Authorizations toAuthorizations(Authorizations authorizations) {
2718     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2719     for (String label : authorizations.getLabels()) {
2720       builder.addLabel(label);
2721     }
2722     return builder.build();
2723   }
2724 
2725   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(String user,
2726       Permission perms) {
2727     return AccessControlProtos.UsersAndPermissions.newBuilder()
2728       .addUserPermissions(AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder()
2729         .setUser(ByteString.copyFromUtf8(user))
2730         .addPermissions(toPermission(perms))
2731         .build())
2732       .build();
2733   }
2734 
2735   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(
2736       ListMultimap<String, Permission> perms) {
2737     AccessControlProtos.UsersAndPermissions.Builder builder =
2738         AccessControlProtos.UsersAndPermissions.newBuilder();
2739     for (Map.Entry<String, Collection<Permission>> entry : perms.asMap().entrySet()) {
2740       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
2741         AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
2742       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
2743       for (Permission perm: entry.getValue()) {
2744         userPermBuilder.addPermissions(toPermission(perm));
2745       }
2746       builder.addUserPermissions(userPermBuilder.build());
2747     }
2748     return builder.build();
2749   }
2750 
2751   public static ListMultimap<String, Permission> toUsersAndPermissions(
2752       AccessControlProtos.UsersAndPermissions proto) {
2753     ListMultimap<String, Permission> result = ArrayListMultimap.create();
2754     for (AccessControlProtos.UsersAndPermissions.UserPermissions userPerms:
2755         proto.getUserPermissionsList()) {
2756       String user = userPerms.getUser().toStringUtf8();
2757       for (AccessControlProtos.Permission perm: userPerms.getPermissionsList()) {
2758         result.put(user, toPermission(perm));
2759       }
2760     }
2761     return result;
2762   }
2763 }