View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.protobuf;
19  
20  
21  import static com.google.protobuf.HBaseZeroCopyByteString.zeroCopyGetBytes;
22  import static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME;
23  
24  import java.io.ByteArrayOutputStream;
25  import java.io.IOException;
26  import java.io.InputStream;
27  import java.lang.reflect.Constructor;
28  import java.lang.reflect.InvocationTargetException;
29  import java.lang.reflect.Method;
30  import java.lang.reflect.ParameterizedType;
31  import java.lang.reflect.Type;
32  import java.nio.ByteBuffer;
33  import java.util.ArrayList;
34  import java.util.Collection;
35  import java.util.HashMap;
36  import java.util.List;
37  import java.util.Map;
38  import java.util.Map.Entry;
39  import java.util.NavigableSet;
40  import java.util.concurrent.TimeUnit;
41  
42  import org.apache.hadoop.conf.Configuration;
43  import org.apache.hadoop.fs.Path;
44  import org.apache.hadoop.hbase.Cell;
45  import org.apache.hadoop.hbase.CellScanner;
46  import org.apache.hadoop.hbase.CellUtil;
47  import org.apache.hadoop.hbase.DoNotRetryIOException;
48  import org.apache.hadoop.hbase.HBaseConfiguration;
49  import org.apache.hadoop.hbase.HConstants;
50  import org.apache.hadoop.hbase.HRegionInfo;
51  import org.apache.hadoop.hbase.HTableDescriptor;
52  import org.apache.hadoop.hbase.KeyValue;
53  import org.apache.hadoop.hbase.NamespaceDescriptor;
54  import org.apache.hadoop.hbase.ServerName;
55  import org.apache.hadoop.hbase.TableName;
56  import org.apache.hadoop.hbase.Tag;
57  import org.apache.hadoop.hbase.classification.InterfaceAudience;
58  import org.apache.hadoop.hbase.client.Append;
59  import org.apache.hadoop.hbase.client.Consistency;
60  import org.apache.hadoop.hbase.client.Delete;
61  import org.apache.hadoop.hbase.client.Durability;
62  import org.apache.hadoop.hbase.client.Get;
63  import org.apache.hadoop.hbase.client.Increment;
64  import org.apache.hadoop.hbase.client.Mutation;
65  import org.apache.hadoop.hbase.client.Put;
66  import org.apache.hadoop.hbase.client.Result;
67  import org.apache.hadoop.hbase.client.Scan;
68  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
69  import org.apache.hadoop.hbase.client.security.SecurityCapability;
70  import org.apache.hadoop.hbase.exceptions.DeserializationException;
71  import org.apache.hadoop.hbase.filter.ByteArrayComparable;
72  import org.apache.hadoop.hbase.filter.Filter;
73  import org.apache.hadoop.hbase.io.LimitInputStream;
74  import org.apache.hadoop.hbase.io.TimeRange;
75  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
76  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService;
77  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
78  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
79  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
80  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
81  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
82  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
83  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
84  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
85  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
86  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
87  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
88  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
89  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
90  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
91  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
92  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest;
93  import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
94  import org.apache.hadoop.hbase.protobuf.generated.CellProtos;
95  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
96  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
97  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
98  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
99  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Column;
100 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall;
101 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
102 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
103 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
104 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
105 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
106 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue;
107 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue.QualifierValue;
108 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.DeleteType;
109 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
110 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
111 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
112 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
113 import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
114 import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
115 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
116 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
117 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo;
118 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
119 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
120 import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
121 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos;
122 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
123 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
124 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService;
125 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos;
126 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
127 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
128 import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
129 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
130 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
131 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
132 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
133 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor.EventType;
134 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.BulkLoadDescriptor;
135 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
136 import org.apache.hadoop.hbase.quotas.QuotaScope;
137 import org.apache.hadoop.hbase.quotas.QuotaType;
138 import org.apache.hadoop.hbase.quotas.ThrottleType;
139 import org.apache.hadoop.hbase.replication.ReplicationLoadSink;
140 import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
141 import org.apache.hadoop.hbase.security.access.Permission;
142 import org.apache.hadoop.hbase.security.access.TablePermission;
143 import org.apache.hadoop.hbase.security.access.UserPermission;
144 import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier;
145 import org.apache.hadoop.hbase.security.visibility.Authorizations;
146 import org.apache.hadoop.hbase.security.visibility.CellVisibility;
147 import org.apache.hadoop.hbase.util.ByteStringer;
148 import org.apache.hadoop.hbase.util.Bytes;
149 import org.apache.hadoop.hbase.util.DynamicClassLoader;
150 import org.apache.hadoop.hbase.util.ExceptionUtil;
151 import org.apache.hadoop.hbase.util.Methods;
152 import org.apache.hadoop.hbase.util.Pair;
153 import org.apache.hadoop.hbase.util.VersionInfo;
154 import org.apache.hadoop.io.Text;
155 import org.apache.hadoop.ipc.RemoteException;
156 import org.apache.hadoop.security.token.Token;
157 
158 import com.google.common.collect.ArrayListMultimap;
159 import com.google.common.collect.ListMultimap;
160 import com.google.common.collect.Lists;
161 import com.google.protobuf.ByteString;
162 import com.google.protobuf.CodedInputStream;
163 import com.google.protobuf.InvalidProtocolBufferException;
164 import com.google.protobuf.Message;
165 import com.google.protobuf.Parser;
166 import com.google.protobuf.RpcChannel;
167 import com.google.protobuf.RpcController;
168 import com.google.protobuf.Service;
169 import com.google.protobuf.ServiceException;
170 import com.google.protobuf.TextFormat;
171 
172 /**
173  * Protobufs utility.
174  */
175 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="DP_CREATE_CLASSLOADER_INSIDE_DO_PRIVILEGED",
176   justification="None. Address sometime.")
177 @InterfaceAudience.Private // TODO: some clients (Hive, etc) use this class
178 public final class ProtobufUtil {
179 
180   private ProtobufUtil() {
181   }
182 
183   /**
184    * Primitive type to class mapping.
185    */
186   private final static Map<String, Class<?>>
187     PRIMITIVES = new HashMap<String, Class<?>>();
188 
189 
190   /**
191    * Many results are simple: no cell, exists true or false. To save on object creations,
192    *  we reuse them across calls.
193    */
194   private final static Cell[] EMPTY_CELL_ARRAY = new Cell[]{};
195   private final static Result EMPTY_RESULT = Result.create(EMPTY_CELL_ARRAY);
196   private final static Result EMPTY_RESULT_EXISTS_TRUE = Result.create(null, true);
197   private final static Result EMPTY_RESULT_EXISTS_FALSE = Result.create(null, false);
198   private final static Result EMPTY_RESULT_STALE = Result.create(EMPTY_CELL_ARRAY, null, true);
199   private final static Result EMPTY_RESULT_EXISTS_TRUE_STALE
200     = Result.create((Cell[])null, true, true);
201   private final static Result EMPTY_RESULT_EXISTS_FALSE_STALE
202     = Result.create((Cell[])null, false, true);
203 
204   private final static ClientProtos.Result EMPTY_RESULT_PB;
205   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_TRUE;
206   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_FALSE;
207   private final static ClientProtos.Result EMPTY_RESULT_PB_STALE;
208   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_TRUE_STALE;
209   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_FALSE_STALE;
210 
211 
212   static {
213     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
214 
215     builder.setExists(true);
216     builder.setAssociatedCellCount(0);
217     EMPTY_RESULT_PB_EXISTS_TRUE =  builder.build();
218 
219     builder.setStale(true);
220     EMPTY_RESULT_PB_EXISTS_TRUE_STALE = builder.build();
221     builder.clear();
222 
223     builder.setExists(false);
224     builder.setAssociatedCellCount(0);
225     EMPTY_RESULT_PB_EXISTS_FALSE =  builder.build();
226     builder.setStale(true);
227     EMPTY_RESULT_PB_EXISTS_FALSE_STALE = builder.build();
228 
229     builder.clear();
230     builder.setAssociatedCellCount(0);
231     EMPTY_RESULT_PB =  builder.build();
232     builder.setStale(true);
233     EMPTY_RESULT_PB_STALE = builder.build();
234   }
235 
236   /**
237    * Dynamic class loader to load filter/comparators
238    */
239   private final static ClassLoader CLASS_LOADER;
240 
241   static {
242     ClassLoader parent = ProtobufUtil.class.getClassLoader();
243     Configuration conf = HBaseConfiguration.create();
244     CLASS_LOADER = new DynamicClassLoader(conf, parent);
245 
246     PRIMITIVES.put(Boolean.TYPE.getName(), Boolean.TYPE);
247     PRIMITIVES.put(Byte.TYPE.getName(), Byte.TYPE);
248     PRIMITIVES.put(Character.TYPE.getName(), Character.TYPE);
249     PRIMITIVES.put(Short.TYPE.getName(), Short.TYPE);
250     PRIMITIVES.put(Integer.TYPE.getName(), Integer.TYPE);
251     PRIMITIVES.put(Long.TYPE.getName(), Long.TYPE);
252     PRIMITIVES.put(Float.TYPE.getName(), Float.TYPE);
253     PRIMITIVES.put(Double.TYPE.getName(), Double.TYPE);
254     PRIMITIVES.put(Void.TYPE.getName(), Void.TYPE);
255   }
256 
257   /**
258    * Magic we put ahead of a serialized protobuf message.
259    * For example, all znode content is protobuf messages with the below magic
260    * for preamble.
261    */
262   public static final byte [] PB_MAGIC = new byte [] {'P', 'B', 'U', 'F'};
263   private static final String PB_MAGIC_STR = Bytes.toString(PB_MAGIC);
264 
265   /**
266    * Prepend the passed bytes with four bytes of magic, {@link #PB_MAGIC}, to flag what
267    * follows as a protobuf in hbase.  Prepend these bytes to all content written to znodes, etc.
268    * @param bytes Bytes to decorate
269    * @return The passed <code>bytes</code> with magic prepended (Creates a new
270    * byte array that is <code>bytes.length</code> plus {@link #PB_MAGIC}.length.
271    */
272   public static byte [] prependPBMagic(final byte [] bytes) {
273     return Bytes.add(PB_MAGIC, bytes);
274   }
275 
276   /**
277    * @param bytes Bytes to check.
278    * @return True if passed <code>bytes</code> has {@link #PB_MAGIC} for a prefix.
279    */
280   public static boolean isPBMagicPrefix(final byte [] bytes) {
281     if (bytes == null) return false;
282     return isPBMagicPrefix(bytes, 0, bytes.length);
283   }
284 
285   /**
286    * @param bytes Bytes to check.
287    * @param offset offset to start at
288    * @param len length to use
289    * @return True if passed <code>bytes</code> has {@link #PB_MAGIC} for a prefix.
290    */
291   public static boolean isPBMagicPrefix(final byte [] bytes, int offset, int len) {
292     if (bytes == null || len < PB_MAGIC.length) return false;
293     return Bytes.compareTo(PB_MAGIC, 0, PB_MAGIC.length, bytes, offset, PB_MAGIC.length) == 0;
294   }
295 
296   /**
297    * @param bytes bytes to check
298    * @throws DeserializationException if we are missing the pb magic prefix
299    */
300   public static void expectPBMagicPrefix(final byte [] bytes) throws DeserializationException {
301     if (!isPBMagicPrefix(bytes)) {
302       throw new DeserializationException("Missing pb magic " + PB_MAGIC_STR + " prefix");
303     }
304   }
305 
306   /**
307    * @return Length of {@link #PB_MAGIC}
308    */
309   public static int lengthOfPBMagic() {
310     return PB_MAGIC.length;
311   }
312 
313   /**
314    * Return the IOException thrown by the remote server wrapped in
315    * ServiceException as cause.
316    *
317    * @param se ServiceException that wraps IO exception thrown by the server
318    * @return Exception wrapped in ServiceException or
319    *   a new IOException that wraps the unexpected ServiceException.
320    */
321   public static IOException getRemoteException(ServiceException se) {
322     Throwable e = se.getCause();
323     if (e == null) {
324       return new IOException(se);
325     }
326     if (ExceptionUtil.isInterrupt(e)) {
327       return ExceptionUtil.asInterrupt(e);
328     }
329     if (e instanceof RemoteException) {
330       e = ((RemoteException) e).unwrapRemoteException();
331     }
332     return e instanceof IOException ? (IOException) e : new IOException(se);
333   }
334 
335   /**
336    * Convert a ServerName to a protocol buffer ServerName
337    *
338    * @param serverName the ServerName to convert
339    * @return the converted protocol buffer ServerName
340    * @see #toServerName(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName)
341    */
342   public static HBaseProtos.ServerName
343       toServerName(final ServerName serverName) {
344     if (serverName == null) return null;
345     HBaseProtos.ServerName.Builder builder =
346       HBaseProtos.ServerName.newBuilder();
347     builder.setHostName(serverName.getHostname());
348     if (serverName.getPort() >= 0) {
349       builder.setPort(serverName.getPort());
350     }
351     if (serverName.getStartcode() >= 0) {
352       builder.setStartCode(serverName.getStartcode());
353     }
354     return builder.build();
355   }
356 
357   /**
358    * Convert a protocol buffer ServerName to a ServerName
359    *
360    * @param proto the protocol buffer ServerName to convert
361    * @return the converted ServerName
362    */
363   public static ServerName toServerName(final HBaseProtos.ServerName proto) {
364     if (proto == null) return null;
365     String hostName = proto.getHostName();
366     long startCode = -1;
367     int port = -1;
368     if (proto.hasPort()) {
369       port = proto.getPort();
370     }
371     if (proto.hasStartCode()) {
372       startCode = proto.getStartCode();
373     }
374     return ServerName.valueOf(hostName, port, startCode);
375   }
376 
377   /**
378    * Get HTableDescriptor[] from GetTableDescriptorsResponse protobuf
379    *
380    * @param proto the GetTableDescriptorsResponse
381    * @return HTableDescriptor[]
382    */
383   public static HTableDescriptor[] getHTableDescriptorArray(GetTableDescriptorsResponse proto) {
384     if (proto == null) return null;
385 
386     HTableDescriptor[] ret = new HTableDescriptor[proto.getTableSchemaCount()];
387     for (int i = 0; i < proto.getTableSchemaCount(); ++i) {
388       ret[i] = HTableDescriptor.convert(proto.getTableSchema(i));
389     }
390     return ret;
391   }
392 
393   /**
394    * get the split keys in form "byte [][]" from a CreateTableRequest proto
395    *
396    * @param proto the CreateTableRequest
397    * @return the split keys
398    */
399   public static byte [][] getSplitKeysArray(final CreateTableRequest proto) {
400     byte [][] splitKeys = new byte[proto.getSplitKeysCount()][];
401     for (int i = 0; i < proto.getSplitKeysCount(); ++i) {
402       splitKeys[i] = proto.getSplitKeys(i).toByteArray();
403     }
404     return splitKeys;
405   }
406 
407   /**
408    * Convert a protobuf Durability into a client Durability
409    */
410   public static Durability toDurability(
411       final ClientProtos.MutationProto.Durability proto) {
412     switch(proto) {
413     case USE_DEFAULT:
414       return Durability.USE_DEFAULT;
415     case SKIP_WAL:
416       return Durability.SKIP_WAL;
417     case ASYNC_WAL:
418       return Durability.ASYNC_WAL;
419     case SYNC_WAL:
420       return Durability.SYNC_WAL;
421     case FSYNC_WAL:
422       return Durability.FSYNC_WAL;
423     default:
424       return Durability.USE_DEFAULT;
425     }
426   }
427 
428   /**
429    * Convert a client Durability into a protbuf Durability
430    */
431   public static ClientProtos.MutationProto.Durability toDurability(
432       final Durability d) {
433     switch(d) {
434     case USE_DEFAULT:
435       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
436     case SKIP_WAL:
437       return ClientProtos.MutationProto.Durability.SKIP_WAL;
438     case ASYNC_WAL:
439       return ClientProtos.MutationProto.Durability.ASYNC_WAL;
440     case SYNC_WAL:
441       return ClientProtos.MutationProto.Durability.SYNC_WAL;
442     case FSYNC_WAL:
443       return ClientProtos.MutationProto.Durability.FSYNC_WAL;
444     default:
445       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
446     }
447   }
448 
449   /**
450    * Convert a protocol buffer Get to a client Get
451    *
452    * @param proto the protocol buffer Get to convert
453    * @return the converted client Get
454    * @throws IOException
455    */
456   public static Get toGet(
457       final ClientProtos.Get proto) throws IOException {
458     if (proto == null) return null;
459     byte[] row = zeroCopyGetBytes(proto.getRow());
460     Get get = new Get(row);
461     if (proto.hasCacheBlocks()) {
462       get.setCacheBlocks(proto.getCacheBlocks());
463     }
464     if (proto.hasMaxVersions()) {
465       get.setMaxVersions(proto.getMaxVersions());
466     }
467     if (proto.hasStoreLimit()) {
468       get.setMaxResultsPerColumnFamily(proto.getStoreLimit());
469     }
470     if (proto.hasStoreOffset()) {
471       get.setRowOffsetPerColumnFamily(proto.getStoreOffset());
472     }
473     if (proto.getCfTimeRangeCount() > 0) {
474       for (HBaseProtos.ColumnFamilyTimeRange cftr : proto.getCfTimeRangeList()) {
475         TimeRange timeRange = protoToTimeRange(cftr.getTimeRange());
476         get.setColumnFamilyTimeRange(cftr.getColumnFamily().toByteArray(),
477             timeRange.getMin(), timeRange.getMax());
478       }
479     }
480     if (proto.hasTimeRange()) {
481       TimeRange timeRange = protoToTimeRange(proto.getTimeRange());
482       get.setTimeRange(timeRange.getMin(), timeRange.getMax());
483     }
484     if (proto.hasFilter()) {
485       FilterProtos.Filter filter = proto.getFilter();
486       get.setFilter(ProtobufUtil.toFilter(filter));
487     }
488     for (NameBytesPair attribute: proto.getAttributeList()) {
489       get.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
490     }
491     if (proto.getColumnCount() > 0) {
492       for (Column column: proto.getColumnList()) {
493         byte[] family = column.getFamily().toByteArray();
494         if (column.getQualifierCount() > 0) {
495           for (ByteString qualifier: column.getQualifierList()) {
496             get.addColumn(family, qualifier.toByteArray());
497           }
498         } else {
499           get.addFamily(family);
500         }
501       }
502     }
503     if (proto.hasExistenceOnly() && proto.getExistenceOnly()){
504       get.setCheckExistenceOnly(true);
505     }
506     if (proto.hasClosestRowBefore() && proto.getClosestRowBefore()){
507       get.setClosestRowBefore(true);
508     }
509     if (proto.hasConsistency()) {
510       get.setConsistency(toConsistency(proto.getConsistency()));
511     }
512     return get;
513   }
514 
515   public static Consistency toConsistency(ClientProtos.Consistency consistency) {
516     switch (consistency) {
517       case STRONG : return Consistency.STRONG;
518       case TIMELINE : return Consistency.TIMELINE;
519       default : return Consistency.STRONG;
520     }
521   }
522 
523   public static ClientProtos.Consistency toConsistency(Consistency consistency) {
524     switch (consistency) {
525       case STRONG : return ClientProtos.Consistency.STRONG;
526       case TIMELINE : return ClientProtos.Consistency.TIMELINE;
527       default : return ClientProtos.Consistency.STRONG;
528     }
529   }
530 
531   /**
532    * Convert a protocol buffer Mutate to a Put.
533    *
534    * @param proto The protocol buffer MutationProto to convert
535    * @return A client Put.
536    * @throws IOException
537    */
538   public static Put toPut(final MutationProto proto)
539   throws IOException {
540     return toPut(proto, null);
541   }
542 
543   /**
544    * Convert a protocol buffer Mutate to a Put.
545    *
546    * @param proto The protocol buffer MutationProto to convert
547    * @param cellScanner If non-null, the Cell data that goes with this proto.
548    * @return A client Put.
549    * @throws IOException
550    */
551   public static Put toPut(final MutationProto proto, final CellScanner cellScanner)
552   throws IOException {
553     // TODO: Server-side at least why do we convert back to the Client types?  Why not just pb it?
554     MutationType type = proto.getMutateType();
555     assert type == MutationType.PUT: type.name();
556     long timestamp = proto.hasTimestamp()? proto.getTimestamp(): HConstants.LATEST_TIMESTAMP;
557     Put put = proto.hasRow() ? new Put(zeroCopyGetBytes(proto.getRow()), timestamp) : null;
558     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
559     if (cellCount > 0) {
560       // The proto has metadata only and the data is separate to be found in the cellScanner.
561       if (cellScanner == null) {
562         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
563             toShortString(proto));
564       }
565       for (int i = 0; i < cellCount; i++) {
566         if (!cellScanner.advance()) {
567           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
568             " no cell returned: " + toShortString(proto));
569         }
570         Cell cell = cellScanner.current();
571         if (put == null) {
572           put = new Put(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
573         }
574         put.add(cell);
575       }
576     } else {
577       if (put == null) {
578         throw new IllegalArgumentException("row cannot be null");
579       }
580       // The proto has the metadata and the data itself
581       for (ColumnValue column: proto.getColumnValueList()) {
582         byte[] family = zeroCopyGetBytes(column.getFamily());
583         for (QualifierValue qv: column.getQualifierValueList()) {
584           if (!qv.hasValue()) {
585             throw new DoNotRetryIOException(
586                 "Missing required field: qualifier value");
587           }
588           ByteBuffer qualifier =
589               qv.hasQualifier() ? qv.getQualifier().asReadOnlyByteBuffer() : null;
590           ByteBuffer value =
591               qv.hasValue() ? qv.getValue().asReadOnlyByteBuffer() : null;
592           long ts = timestamp;
593           if (qv.hasTimestamp()) {
594             ts = qv.getTimestamp();
595           }
596           byte[] tags;
597           if (qv.hasTags()) {
598             tags = zeroCopyGetBytes(qv.getTags());
599             Object[] array = Tag.asList(tags, 0, (short)tags.length).toArray();
600             Tag[] tagArray = new Tag[array.length];
601             for(int i = 0; i< array.length; i++) {
602               tagArray[i] = (Tag)array[i];
603             }
604             if(qv.hasDeleteType()) {
605               byte[] qual = qv.hasQualifier() ? zeroCopyGetBytes(qv.getQualifier()) : null;
606               put.add(new KeyValue(zeroCopyGetBytes(proto.getRow()), family, qual, ts,
607                   fromDeleteType(qv.getDeleteType()), null, tags));
608             } else {
609               put.addImmutable(family, qualifier, ts, value, tagArray);
610             }
611           } else {
612             if(qv.hasDeleteType()) {
613               byte[] qual = qv.hasQualifier() ? zeroCopyGetBytes(qv.getQualifier()) : null;
614               put.add(new KeyValue(zeroCopyGetBytes(proto.getRow()), family, qual, ts,
615                   fromDeleteType(qv.getDeleteType())));
616             } else{
617               put.addImmutable(family, qualifier, ts, value);
618             }
619           }
620         }
621       }
622     }
623     put.setDurability(toDurability(proto.getDurability()));
624     for (NameBytesPair attribute: proto.getAttributeList()) {
625       put.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
626     }
627     return put;
628   }
629 
630   /**
631    * Convert a protocol buffer Mutate to a Delete
632    *
633    * @param proto the protocol buffer Mutate to convert
634    * @return the converted client Delete
635    * @throws IOException
636    */
637   public static Delete toDelete(final MutationProto proto)
638   throws IOException {
639     return toDelete(proto, null);
640   }
641 
642   /**
643    * Convert a protocol buffer Mutate to a Delete
644    *
645    * @param proto the protocol buffer Mutate to convert
646    * @param cellScanner if non-null, the data that goes with this delete.
647    * @return the converted client Delete
648    * @throws IOException
649    */
650   public static Delete toDelete(final MutationProto proto, final CellScanner cellScanner)
651   throws IOException {
652     MutationType type = proto.getMutateType();
653     assert type == MutationType.DELETE : type.name();
654     long timestamp = proto.hasTimestamp() ? proto.getTimestamp() : HConstants.LATEST_TIMESTAMP;
655     Delete delete = proto.hasRow() ? new Delete(zeroCopyGetBytes(proto.getRow()), timestamp) : null;
656     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
657     if (cellCount > 0) {
658       // The proto has metadata only and the data is separate to be found in the cellScanner.
659       if (cellScanner == null) {
660         // TextFormat should be fine for a Delete since it carries no data, just coordinates.
661         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
662           TextFormat.shortDebugString(proto));
663       }
664       for (int i = 0; i < cellCount; i++) {
665         if (!cellScanner.advance()) {
666           // TextFormat should be fine for a Delete since it carries no data, just coordinates.
667           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
668             " no cell returned: " + TextFormat.shortDebugString(proto));
669         }
670         Cell cell = cellScanner.current();
671         if (delete == null) {
672           delete =
673             new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
674         }
675         delete.addDeleteMarker(cell);
676       }
677     } else {
678       if (delete == null) {
679         throw new IllegalArgumentException("row cannot be null");
680       }
681       for (ColumnValue column: proto.getColumnValueList()) {
682         byte[] family = column.getFamily().toByteArray();
683         for (QualifierValue qv: column.getQualifierValueList()) {
684           DeleteType deleteType = qv.getDeleteType();
685           byte[] qualifier = null;
686           if (qv.hasQualifier()) {
687             qualifier = qv.getQualifier().toByteArray();
688           }
689           long ts = HConstants.LATEST_TIMESTAMP;
690           if (qv.hasTimestamp()) {
691             ts = qv.getTimestamp();
692           }
693           if (deleteType == DeleteType.DELETE_ONE_VERSION) {
694             delete.deleteColumn(family, qualifier, ts);
695           } else if (deleteType == DeleteType.DELETE_MULTIPLE_VERSIONS) {
696             delete.deleteColumns(family, qualifier, ts);
697           } else if (deleteType == DeleteType.DELETE_FAMILY_VERSION) {
698             delete.deleteFamilyVersion(family, ts);
699           } else {
700             delete.deleteFamily(family, ts);
701           }
702         }
703       }
704     }
705     delete.setDurability(toDurability(proto.getDurability()));
706     for (NameBytesPair attribute: proto.getAttributeList()) {
707       delete.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
708     }
709     return delete;
710   }
711 
712   /**
713    * Convert a protocol buffer Mutate to an Append
714    * @param cellScanner
715    * @param proto the protocol buffer Mutate to convert
716    * @return the converted client Append
717    * @throws IOException
718    */
719   public static Append toAppend(final MutationProto proto, final CellScanner cellScanner)
720   throws IOException {
721     MutationType type = proto.getMutateType();
722     assert type == MutationType.APPEND : type.name();
723     byte [] row = proto.hasRow()? zeroCopyGetBytes(proto.getRow()): null;
724     Append append = null;
725     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
726     if (cellCount > 0) {
727       // The proto has metadata only and the data is separate to be found in the cellScanner.
728       if (cellScanner == null) {
729         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
730           toShortString(proto));
731       }
732       for (int i = 0; i < cellCount; i++) {
733         if (!cellScanner.advance()) {
734           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
735             " no cell returned: " + toShortString(proto));
736         }
737         Cell cell = cellScanner.current();
738         if (append == null) {
739           append = new Append(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
740         }
741         append.add(cell);
742       }
743     } else {
744       append = new Append(row);
745       for (ColumnValue column: proto.getColumnValueList()) {
746         byte[] family = zeroCopyGetBytes(column.getFamily());
747         for (QualifierValue qv: column.getQualifierValueList()) {
748           byte[] qualifier = zeroCopyGetBytes(qv.getQualifier());
749           if (!qv.hasValue()) {
750             throw new DoNotRetryIOException(
751               "Missing required field: qualifier value");
752           }
753           byte[] value = zeroCopyGetBytes(qv.getValue());
754           byte[] tags = null;
755           if (qv.hasTags()) {
756             tags = zeroCopyGetBytes(qv.getTags());
757           }
758           append.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
759               KeyValue.Type.Put, value, tags));
760         }
761       }
762     }
763     append.setDurability(toDurability(proto.getDurability()));
764     for (NameBytesPair attribute: proto.getAttributeList()) {
765       append.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
766     }
767     return append;
768   }
769 
770   /**
771    * Convert a MutateRequest to Mutation
772    *
773    * @param proto the protocol buffer Mutate to convert
774    * @return the converted Mutation
775    * @throws IOException
776    */
777   public static Mutation toMutation(final MutationProto proto) throws IOException {
778     MutationType type = proto.getMutateType();
779     if (type == MutationType.APPEND) {
780       return toAppend(proto, null);
781     }
782     if (type == MutationType.DELETE) {
783       return toDelete(proto, null);
784     }
785     if (type == MutationType.PUT) {
786       return toPut(proto, null);
787     }
788     throw new IOException("Unknown mutation type " + type);
789   }
790 
791   /**
792    * Convert a protocol buffer Mutate to an Increment
793    *
794    * @param proto the protocol buffer Mutate to convert
795    * @return the converted client Increment
796    * @throws IOException
797    */
798   public static Increment toIncrement(final MutationProto proto, final CellScanner cellScanner)
799   throws IOException {
800     MutationType type = proto.getMutateType();
801     assert type == MutationType.INCREMENT : type.name();
802     byte [] row = proto.hasRow()? zeroCopyGetBytes(proto.getRow()): null;
803     Increment increment = null;
804     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
805     if (cellCount > 0) {
806       // The proto has metadata only and the data is separate to be found in the cellScanner.
807       if (cellScanner == null) {
808         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
809           TextFormat.shortDebugString(proto));
810       }
811       for (int i = 0; i < cellCount; i++) {
812         if (!cellScanner.advance()) {
813           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
814             " no cell returned: " + TextFormat.shortDebugString(proto));
815         }
816         Cell cell = cellScanner.current();
817         if (increment == null) {
818           increment = new Increment(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
819         }
820         increment.add(cell);
821       }
822     } else {
823       increment = new Increment(row);
824       for (ColumnValue column: proto.getColumnValueList()) {
825         byte[] family = zeroCopyGetBytes(column.getFamily());
826         for (QualifierValue qv: column.getQualifierValueList()) {
827           byte[] qualifier = zeroCopyGetBytes(qv.getQualifier());
828           if (!qv.hasValue()) {
829             throw new DoNotRetryIOException("Missing required field: qualifier value");
830           }
831           byte[] value = zeroCopyGetBytes(qv.getValue());
832           byte[] tags = null;
833           if (qv.hasTags()) {
834             tags = zeroCopyGetBytes(qv.getTags());
835           }
836           increment.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
837               KeyValue.Type.Put, value, tags));
838         }
839       }
840     }
841     if (proto.hasTimeRange()) {
842       TimeRange timeRange = protoToTimeRange(proto.getTimeRange());
843       increment.setTimeRange(timeRange.getMin(), timeRange.getMax());
844     }
845     increment.setDurability(toDurability(proto.getDurability()));
846     for (NameBytesPair attribute : proto.getAttributeList()) {
847       increment.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
848     }
849     return increment;
850   }
851 
852   /**
853    * Convert a client Scan to a protocol buffer Scan
854    *
855    * @param scan the client Scan to convert
856    * @return the converted protocol buffer Scan
857    * @throws IOException
858    */
859   public static ClientProtos.Scan toScan(
860       final Scan scan) throws IOException {
861     ClientProtos.Scan.Builder scanBuilder =
862       ClientProtos.Scan.newBuilder();
863     scanBuilder.setCacheBlocks(scan.getCacheBlocks());
864     if (scan.getBatch() > 0) {
865       scanBuilder.setBatchSize(scan.getBatch());
866     }
867     if (scan.getMaxResultSize() > 0) {
868       scanBuilder.setMaxResultSize(scan.getMaxResultSize());
869     }
870     if (scan.isSmall()) {
871       scanBuilder.setSmall(scan.isSmall());
872     }
873     if (scan.getAllowPartialResults()) {
874       scanBuilder.setAllowPartialResults(scan.getAllowPartialResults());
875     }
876     Boolean loadColumnFamiliesOnDemand = scan.getLoadColumnFamiliesOnDemandValue();
877     if (loadColumnFamiliesOnDemand != null) {
878       scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand.booleanValue());
879     }
880     scanBuilder.setMaxVersions(scan.getMaxVersions());
881     for (Entry<byte[], TimeRange> cftr : scan.getColumnFamilyTimeRange().entrySet()) {
882       HBaseProtos.ColumnFamilyTimeRange.Builder b = HBaseProtos.ColumnFamilyTimeRange.newBuilder();
883       b.setColumnFamily(ByteString.copyFrom(cftr.getKey()));
884       b.setTimeRange(timeRangeToProto(cftr.getValue()));
885       scanBuilder.addCfTimeRange(b);
886     }
887     TimeRange timeRange = scan.getTimeRange();
888     if (!timeRange.isAllTime()) {
889       HBaseProtos.TimeRange.Builder timeRangeBuilder =
890         HBaseProtos.TimeRange.newBuilder();
891       timeRangeBuilder.setFrom(timeRange.getMin());
892       timeRangeBuilder.setTo(timeRange.getMax());
893       scanBuilder.setTimeRange(timeRangeBuilder.build());
894     }
895     Map<String, byte[]> attributes = scan.getAttributesMap();
896     if (!attributes.isEmpty()) {
897       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
898       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
899         attributeBuilder.setName(attribute.getKey());
900         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
901         scanBuilder.addAttribute(attributeBuilder.build());
902       }
903     }
904     byte[] startRow = scan.getStartRow();
905     if (startRow != null && startRow.length > 0) {
906       scanBuilder.setStartRow(ByteStringer.wrap(startRow));
907     }
908     byte[] stopRow = scan.getStopRow();
909     if (stopRow != null && stopRow.length > 0) {
910       scanBuilder.setStopRow(ByteStringer.wrap(stopRow));
911     }
912     if (scan.hasFilter()) {
913       scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
914     }
915     if (scan.hasFamilies()) {
916       Column.Builder columnBuilder = Column.newBuilder();
917       for (Map.Entry<byte[],NavigableSet<byte []>>
918           family: scan.getFamilyMap().entrySet()) {
919         columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
920         NavigableSet<byte []> qualifiers = family.getValue();
921         columnBuilder.clearQualifier();
922         if (qualifiers != null && qualifiers.size() > 0) {
923           for (byte [] qualifier: qualifiers) {
924             columnBuilder.addQualifier(ByteStringer.wrap(qualifier));
925           }
926         }
927         scanBuilder.addColumn(columnBuilder.build());
928       }
929     }
930     if (scan.getMaxResultsPerColumnFamily() >= 0) {
931       scanBuilder.setStoreLimit(scan.getMaxResultsPerColumnFamily());
932     }
933     if (scan.getRowOffsetPerColumnFamily() > 0) {
934       scanBuilder.setStoreOffset(scan.getRowOffsetPerColumnFamily());
935     }
936     if (scan.isReversed()) {
937       scanBuilder.setReversed(scan.isReversed());
938     }
939     if (scan.getConsistency() == Consistency.TIMELINE) {
940       scanBuilder.setConsistency(toConsistency(scan.getConsistency()));
941     }
942     if (scan.getCaching() > 0) {
943       scanBuilder.setCaching(scan.getCaching());
944     }
945     return scanBuilder.build();
946   }
947 
948   /**
949    * Convert a protocol buffer Scan to a client Scan
950    *
951    * @param proto the protocol buffer Scan to convert
952    * @return the converted client Scan
953    * @throws IOException
954    */
955   public static Scan toScan(
956       final ClientProtos.Scan proto) throws IOException {
957     byte [] startRow = HConstants.EMPTY_START_ROW;
958     byte [] stopRow  = HConstants.EMPTY_END_ROW;
959     if (proto.hasStartRow()) {
960       startRow = proto.getStartRow().toByteArray();
961     }
962     if (proto.hasStopRow()) {
963       stopRow = proto.getStopRow().toByteArray();
964     }
965     Scan scan = new Scan(startRow, stopRow);
966     if (proto.hasCacheBlocks()) {
967       scan.setCacheBlocks(proto.getCacheBlocks());
968     }
969     if (proto.hasMaxVersions()) {
970       scan.setMaxVersions(proto.getMaxVersions());
971     }
972     if (proto.hasStoreLimit()) {
973       scan.setMaxResultsPerColumnFamily(proto.getStoreLimit());
974     }
975     if (proto.hasStoreOffset()) {
976       scan.setRowOffsetPerColumnFamily(proto.getStoreOffset());
977     }
978     if (proto.hasLoadColumnFamiliesOnDemand()) {
979       scan.setLoadColumnFamiliesOnDemand(proto.getLoadColumnFamiliesOnDemand());
980     }
981     if (proto.getCfTimeRangeCount() > 0) {
982       for (HBaseProtos.ColumnFamilyTimeRange cftr : proto.getCfTimeRangeList()) {
983         TimeRange timeRange = protoToTimeRange(cftr.getTimeRange());
984         scan.setColumnFamilyTimeRange(cftr.getColumnFamily().toByteArray(),
985             timeRange.getMin(), timeRange.getMax());
986       }
987     }
988     if (proto.hasTimeRange()) {
989       TimeRange timeRange = protoToTimeRange(proto.getTimeRange());
990       scan.setTimeRange(timeRange.getMin(), timeRange.getMax());
991     }
992     if (proto.hasFilter()) {
993       FilterProtos.Filter filter = proto.getFilter();
994       scan.setFilter(ProtobufUtil.toFilter(filter));
995     }
996     if (proto.hasBatchSize()) {
997       scan.setBatch(proto.getBatchSize());
998     }
999     if (proto.hasMaxResultSize()) {
1000       scan.setMaxResultSize(proto.getMaxResultSize());
1001     }
1002     if (proto.hasSmall()) {
1003       scan.setSmall(proto.getSmall());
1004     }
1005     if (proto.hasAllowPartialResults()) {
1006       scan.setAllowPartialResults(proto.getAllowPartialResults());
1007     }
1008     for (NameBytesPair attribute: proto.getAttributeList()) {
1009       scan.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
1010     }
1011     if (proto.getColumnCount() > 0) {
1012       for (Column column: proto.getColumnList()) {
1013         byte[] family = column.getFamily().toByteArray();
1014         if (column.getQualifierCount() > 0) {
1015           for (ByteString qualifier: column.getQualifierList()) {
1016             scan.addColumn(family, qualifier.toByteArray());
1017           }
1018         } else {
1019           scan.addFamily(family);
1020         }
1021       }
1022     }
1023     if (proto.hasReversed()) {
1024       scan.setReversed(proto.getReversed());
1025     }
1026     if (proto.hasConsistency()) {
1027       scan.setConsistency(toConsistency(proto.getConsistency()));
1028     }
1029     if (proto.hasCaching()) {
1030       scan.setCaching(proto.getCaching());
1031     }
1032     return scan;
1033   }
1034 
1035   /**
1036    * Create a protocol buffer Get based on a client Get.
1037    *
1038    * @param get the client Get
1039    * @return a protocol buffer Get
1040    * @throws IOException
1041    */
1042   public static ClientProtos.Get toGet(
1043       final Get get) throws IOException {
1044     ClientProtos.Get.Builder builder =
1045       ClientProtos.Get.newBuilder();
1046     builder.setRow(ByteStringer.wrap(get.getRow()));
1047     builder.setCacheBlocks(get.getCacheBlocks());
1048     builder.setMaxVersions(get.getMaxVersions());
1049     if (get.getFilter() != null) {
1050       builder.setFilter(ProtobufUtil.toFilter(get.getFilter()));
1051     }
1052     for (Entry<byte[], TimeRange> cftr : get.getColumnFamilyTimeRange().entrySet()) {
1053       HBaseProtos.ColumnFamilyTimeRange.Builder b = HBaseProtos.ColumnFamilyTimeRange.newBuilder();
1054       b.setColumnFamily(ByteString.copyFrom(cftr.getKey()));
1055       b.setTimeRange(timeRangeToProto(cftr.getValue()));
1056       builder.addCfTimeRange(b);
1057     }
1058     TimeRange timeRange = get.getTimeRange();
1059     if (!timeRange.isAllTime()) {
1060       HBaseProtos.TimeRange.Builder timeRangeBuilder =
1061         HBaseProtos.TimeRange.newBuilder();
1062       timeRangeBuilder.setFrom(timeRange.getMin());
1063       timeRangeBuilder.setTo(timeRange.getMax());
1064       builder.setTimeRange(timeRangeBuilder.build());
1065     }
1066     Map<String, byte[]> attributes = get.getAttributesMap();
1067     if (!attributes.isEmpty()) {
1068       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1069       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
1070         attributeBuilder.setName(attribute.getKey());
1071         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1072         builder.addAttribute(attributeBuilder.build());
1073       }
1074     }
1075     if (get.hasFamilies()) {
1076       Column.Builder columnBuilder = Column.newBuilder();
1077       Map<byte[], NavigableSet<byte[]>> families = get.getFamilyMap();
1078       for (Map.Entry<byte[], NavigableSet<byte[]>> family: families.entrySet()) {
1079         NavigableSet<byte[]> qualifiers = family.getValue();
1080         columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1081         columnBuilder.clearQualifier();
1082         if (qualifiers != null && qualifiers.size() > 0) {
1083           for (byte[] qualifier: qualifiers) {
1084             columnBuilder.addQualifier(ByteStringer.wrap(qualifier));
1085           }
1086         }
1087         builder.addColumn(columnBuilder.build());
1088       }
1089     }
1090     if (get.getMaxResultsPerColumnFamily() >= 0) {
1091       builder.setStoreLimit(get.getMaxResultsPerColumnFamily());
1092     }
1093     if (get.getRowOffsetPerColumnFamily() > 0) {
1094       builder.setStoreOffset(get.getRowOffsetPerColumnFamily());
1095     }
1096     if (get.isCheckExistenceOnly()){
1097       builder.setExistenceOnly(true);
1098     }
1099     if (get.isClosestRowBefore()){
1100       builder.setClosestRowBefore(true);
1101     }
1102     if (get.getConsistency() != null && get.getConsistency() != Consistency.STRONG) {
1103       builder.setConsistency(toConsistency(get.getConsistency()));
1104     }
1105 
1106     return builder.build();
1107   }
1108 
1109   static void setTimeRange(final MutationProto.Builder builder, final TimeRange timeRange) {
1110     if (!timeRange.isAllTime()) {
1111       HBaseProtos.TimeRange.Builder timeRangeBuilder =
1112         HBaseProtos.TimeRange.newBuilder();
1113       timeRangeBuilder.setFrom(timeRange.getMin());
1114       timeRangeBuilder.setTo(timeRange.getMax());
1115       builder.setTimeRange(timeRangeBuilder.build());
1116     }
1117   }
1118 
1119   /**
1120    * Convert a client Increment to a protobuf Mutate.
1121    *
1122    * @param increment
1123    * @return the converted mutate
1124    */
1125   public static MutationProto toMutation(
1126     final Increment increment, final MutationProto.Builder builder, long nonce) {
1127     builder.setRow(ByteStringer.wrap(increment.getRow()));
1128     builder.setMutateType(MutationType.INCREMENT);
1129     builder.setDurability(toDurability(increment.getDurability()));
1130     if (nonce != HConstants.NO_NONCE) {
1131       builder.setNonce(nonce);
1132     }
1133     TimeRange timeRange = increment.getTimeRange();
1134     setTimeRange(builder, timeRange);
1135     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1136     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1137     for (Map.Entry<byte[], List<Cell>> family: increment.getFamilyCellMap().entrySet()) {
1138       columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1139       columnBuilder.clearQualifierValue();
1140       List<Cell> values = family.getValue();
1141       if (values != null && values.size() > 0) {
1142         for (Cell cell: values) {
1143           valueBuilder.clear();
1144           valueBuilder.setQualifier(ByteStringer.wrap(
1145               cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
1146           valueBuilder.setValue(ByteStringer.wrap(
1147               cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
1148           if (cell.getTagsLength() > 0) {
1149             valueBuilder.setTags(ByteStringer.wrap(cell.getTagsArray(),
1150                 cell.getTagsOffset(), cell.getTagsLength()));
1151           }
1152           columnBuilder.addQualifierValue(valueBuilder.build());
1153         }
1154       }
1155       builder.addColumnValue(columnBuilder.build());
1156     }
1157     Map<String, byte[]> attributes = increment.getAttributesMap();
1158     if (!attributes.isEmpty()) {
1159       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1160       for (Map.Entry<String, byte[]> attribute : attributes.entrySet()) {
1161         attributeBuilder.setName(attribute.getKey());
1162         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1163         builder.addAttribute(attributeBuilder.build());
1164       }
1165     }
1166     return builder.build();
1167   }
1168 
1169   public static MutationProto toMutation(final MutationType type, final Mutation mutation)
1170     throws IOException {
1171     return toMutation(type, mutation, HConstants.NO_NONCE);
1172   }
1173 
1174   /**
1175    * Create a protocol buffer Mutate based on a client Mutation
1176    *
1177    * @param type
1178    * @param mutation
1179    * @return a protobuf'd Mutation
1180    * @throws IOException
1181    */
1182   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1183     final long nonce) throws IOException {
1184     return toMutation(type, mutation, MutationProto.newBuilder(), nonce);
1185   }
1186 
1187   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1188       MutationProto.Builder builder) throws IOException {
1189     return toMutation(type, mutation, builder, HConstants.NO_NONCE);
1190   }
1191 
1192   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1193       MutationProto.Builder builder, long nonce)
1194   throws IOException {
1195     builder = getMutationBuilderAndSetCommonFields(type, mutation, builder);
1196     if (nonce != HConstants.NO_NONCE) {
1197       builder.setNonce(nonce);
1198     }
1199     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1200     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1201     for (Map.Entry<byte[],List<Cell>> family: mutation.getFamilyCellMap().entrySet()) {
1202       columnBuilder.clear();
1203       columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1204       for (Cell cell: family.getValue()) {
1205         valueBuilder.clear();
1206         valueBuilder.setQualifier(ByteStringer.wrap(
1207             cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
1208         valueBuilder.setValue(ByteStringer.wrap(
1209             cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
1210         valueBuilder.setTimestamp(cell.getTimestamp());
1211         if (type == MutationType.DELETE || (type == MutationType.PUT && CellUtil.isDelete(cell))) {
1212           KeyValue.Type keyValueType = KeyValue.Type.codeToType(cell.getTypeByte());
1213           valueBuilder.setDeleteType(toDeleteType(keyValueType));
1214         }
1215         columnBuilder.addQualifierValue(valueBuilder.build());
1216       }
1217       builder.addColumnValue(columnBuilder.build());
1218     }
1219     return builder.build();
1220   }
1221 
1222   /**
1223    * Create a protocol buffer MutationProto based on a client Mutation. Does NOT include data.
1224    * Understanding is that the Cell will be transported other than via protobuf.
1225    * @param type
1226    * @param mutation
1227    * @param builder
1228    * @return a protobuf'd Mutation
1229    * @throws IOException
1230    */
1231   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1232       final MutationProto.Builder builder)  throws IOException {
1233     return toMutationNoData(type, mutation, builder, HConstants.NO_NONCE);
1234   }
1235 
1236   /**
1237    * Create a protocol buffer MutationProto based on a client Mutation.  Does NOT include data.
1238    * Understanding is that the Cell will be transported other than via protobuf.
1239    * @param type
1240    * @param mutation
1241    * @return a protobuf'd Mutation
1242    * @throws IOException
1243    */
1244   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation)
1245   throws IOException {
1246     MutationProto.Builder builder =  MutationProto.newBuilder();
1247     return toMutationNoData(type, mutation, builder);
1248   }
1249 
1250   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1251       final MutationProto.Builder builder, long nonce) throws IOException {
1252     getMutationBuilderAndSetCommonFields(type, mutation, builder);
1253     builder.setAssociatedCellCount(mutation.size());
1254     if (mutation instanceof Increment) {
1255       setTimeRange(builder, ((Increment)mutation).getTimeRange());
1256     }
1257     if (nonce != HConstants.NO_NONCE) {
1258       builder.setNonce(nonce);
1259     }
1260     return builder.build();
1261   }
1262 
1263   /**
1264    * Code shared by {@link #toMutation(MutationType, Mutation)} and
1265    * {@link #toMutationNoData(MutationType, Mutation)}
1266    * @param type
1267    * @param mutation
1268    * @return A partly-filled out protobuf'd Mutation.
1269    */
1270   private static MutationProto.Builder getMutationBuilderAndSetCommonFields(final MutationType type,
1271       final Mutation mutation, MutationProto.Builder builder) {
1272     builder.setRow(ByteStringer.wrap(mutation.getRow()));
1273     builder.setMutateType(type);
1274     builder.setDurability(toDurability(mutation.getDurability()));
1275     builder.setTimestamp(mutation.getTimeStamp());
1276     Map<String, byte[]> attributes = mutation.getAttributesMap();
1277     if (!attributes.isEmpty()) {
1278       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1279       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
1280         attributeBuilder.setName(attribute.getKey());
1281         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1282         builder.addAttribute(attributeBuilder.build());
1283       }
1284     }
1285     return builder;
1286   }
1287 
1288   /**
1289    * Convert a client Result to a protocol buffer Result
1290    *
1291    * @param result the client Result to convert
1292    * @return the converted protocol buffer Result
1293    */
1294   public static ClientProtos.Result toResult(final Result result) {
1295     if (result.getExists() != null) {
1296       return toResult(result.getExists(), result.isStale());
1297     }
1298 
1299     Cell[] cells = result.rawCells();
1300     if (cells == null || cells.length == 0) {
1301       return result.isStale() ? EMPTY_RESULT_PB_STALE : EMPTY_RESULT_PB;
1302     }
1303 
1304     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1305     for (Cell c : cells) {
1306       builder.addCell(toCell(c));
1307     }
1308 
1309     builder.setStale(result.isStale());
1310     builder.setPartial(result.isPartial());
1311 
1312     return builder.build();
1313   }
1314 
1315   /**
1316    * Convert a client Result to a protocol buffer Result
1317    *
1318    * @param existence the client existence to send
1319    * @return the converted protocol buffer Result
1320    */
1321   public static ClientProtos.Result toResult(final boolean existence, boolean stale) {
1322     if (stale){
1323       return existence ? EMPTY_RESULT_PB_EXISTS_TRUE_STALE : EMPTY_RESULT_PB_EXISTS_FALSE_STALE;
1324     } else {
1325       return existence ? EMPTY_RESULT_PB_EXISTS_TRUE : EMPTY_RESULT_PB_EXISTS_FALSE;
1326     }
1327   }
1328 
1329   /**
1330    * Convert a client Result to a protocol buffer Result.
1331    * The pb Result does not include the Cell data.  That is for transport otherwise.
1332    *
1333    * @param result the client Result to convert
1334    * @return the converted protocol buffer Result
1335    */
1336   public static ClientProtos.Result toResultNoData(final Result result) {
1337     if (result.getExists() != null) return toResult(result.getExists(), result.isStale());
1338     int size = result.size();
1339     if (size == 0) return result.isStale() ? EMPTY_RESULT_PB_STALE : EMPTY_RESULT_PB;
1340     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1341     builder.setAssociatedCellCount(size);
1342     builder.setStale(result.isStale());
1343     return builder.build();
1344   }
1345 
1346   /**
1347    * Convert a protocol buffer Result to a client Result
1348    *
1349    * @param proto the protocol buffer Result to convert
1350    * @return the converted client Result
1351    */
1352   public static Result toResult(final ClientProtos.Result proto) {
1353     if (proto.hasExists()) {
1354       if (proto.getStale()) {
1355         return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE_STALE :EMPTY_RESULT_EXISTS_FALSE_STALE;
1356       }
1357       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1358     }
1359 
1360     List<CellProtos.Cell> values = proto.getCellList();
1361     if (values.isEmpty()){
1362       return proto.getStale() ? EMPTY_RESULT_STALE : EMPTY_RESULT;
1363     }
1364 
1365     List<Cell> cells = new ArrayList<Cell>(values.size());
1366     for (CellProtos.Cell c : values) {
1367       cells.add(toCell(c));
1368     }
1369     return Result.create(cells, null, proto.getStale(), proto.getPartial());
1370   }
1371 
1372   /**
1373    * Convert a protocol buffer Result to a client Result
1374    *
1375    * @param proto the protocol buffer Result to convert
1376    * @param scanner Optional cell scanner.
1377    * @return the converted client Result
1378    * @throws IOException
1379    */
1380   public static Result toResult(final ClientProtos.Result proto, final CellScanner scanner)
1381   throws IOException {
1382     List<CellProtos.Cell> values = proto.getCellList();
1383 
1384     if (proto.hasExists()) {
1385       if ((values != null && !values.isEmpty()) ||
1386           (proto.hasAssociatedCellCount() && proto.getAssociatedCellCount() > 0)) {
1387         throw new IllegalArgumentException("bad proto: exists with cells is no allowed " + proto);
1388       }
1389       if (proto.getStale()) {
1390         return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE_STALE :EMPTY_RESULT_EXISTS_FALSE_STALE;
1391       }
1392       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1393     }
1394 
1395     // TODO: Unit test that has some Cells in scanner and some in the proto.
1396     List<Cell> cells = null;
1397     if (proto.hasAssociatedCellCount()) {
1398       int count = proto.getAssociatedCellCount();
1399       cells = new ArrayList<Cell>(count + values.size());
1400       for (int i = 0; i < count; i++) {
1401         if (!scanner.advance()) throw new IOException("Failed get " + i + " of " + count);
1402         cells.add(scanner.current());
1403       }
1404     }
1405 
1406     if (!values.isEmpty()){
1407       if (cells == null) cells = new ArrayList<Cell>(values.size());
1408       for (CellProtos.Cell c: values) {
1409         cells.add(toCell(c));
1410       }
1411     }
1412 
1413     return (cells == null || cells.isEmpty())
1414         ? (proto.getStale() ? EMPTY_RESULT_STALE : EMPTY_RESULT)
1415         : Result.create(cells, null, proto.getStale());
1416   }
1417 
1418 
1419   /**
1420    * Convert a ByteArrayComparable to a protocol buffer Comparator
1421    *
1422    * @param comparator the ByteArrayComparable to convert
1423    * @return the converted protocol buffer Comparator
1424    */
1425   public static ComparatorProtos.Comparator toComparator(ByteArrayComparable comparator) {
1426     ComparatorProtos.Comparator.Builder builder = ComparatorProtos.Comparator.newBuilder();
1427     builder.setName(comparator.getClass().getName());
1428     builder.setSerializedComparator(ByteStringer.wrap(comparator.toByteArray()));
1429     return builder.build();
1430   }
1431 
1432   /**
1433    * Convert a protocol buffer Comparator to a ByteArrayComparable
1434    *
1435    * @param proto the protocol buffer Comparator to convert
1436    * @return the converted ByteArrayComparable
1437    */
1438   @SuppressWarnings("unchecked")
1439   public static ByteArrayComparable toComparator(ComparatorProtos.Comparator proto)
1440   throws IOException {
1441     String type = proto.getName();
1442     String funcName = "parseFrom";
1443     byte [] value = proto.getSerializedComparator().toByteArray();
1444     try {
1445       Class<? extends ByteArrayComparable> c =
1446         (Class<? extends ByteArrayComparable>)Class.forName(type, true, CLASS_LOADER);
1447       Method parseFrom = c.getMethod(funcName, byte[].class);
1448       if (parseFrom == null) {
1449         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1450       }
1451       return (ByteArrayComparable)parseFrom.invoke(null, value);
1452     } catch (Exception e) {
1453       throw new IOException(e);
1454     }
1455   }
1456 
1457   /**
1458    * Convert a protocol buffer Filter to a client Filter
1459    *
1460    * @param proto the protocol buffer Filter to convert
1461    * @return the converted Filter
1462    */
1463   @SuppressWarnings("unchecked")
1464   public static Filter toFilter(FilterProtos.Filter proto) throws IOException {
1465     String type = proto.getName();
1466     final byte [] value = proto.getSerializedFilter().toByteArray();
1467     String funcName = "parseFrom";
1468     try {
1469       Class<? extends Filter> c =
1470         (Class<? extends Filter>)Class.forName(type, true, CLASS_LOADER);
1471       Method parseFrom = c.getMethod(funcName, byte[].class);
1472       if (parseFrom == null) {
1473         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1474       }
1475       return (Filter)parseFrom.invoke(c, value);
1476     } catch (Exception e) {
1477       // Either we couldn't instantiate the method object, or "parseFrom" failed.
1478       // In either case, let's not retry.
1479       throw new DoNotRetryIOException(e);
1480     }
1481   }
1482 
1483   /**
1484    * Convert a client Filter to a protocol buffer Filter
1485    *
1486    * @param filter the Filter to convert
1487    * @return the converted protocol buffer Filter
1488    */
1489   public static FilterProtos.Filter toFilter(Filter filter) throws IOException {
1490     FilterProtos.Filter.Builder builder = FilterProtos.Filter.newBuilder();
1491     builder.setName(filter.getClass().getName());
1492     builder.setSerializedFilter(ByteStringer.wrap(filter.toByteArray()));
1493     return builder.build();
1494   }
1495 
1496   /**
1497    * Convert a delete KeyValue type to protocol buffer DeleteType.
1498    *
1499    * @param type
1500    * @return protocol buffer DeleteType
1501    * @throws IOException
1502    */
1503   public static DeleteType toDeleteType(
1504       KeyValue.Type type) throws IOException {
1505     switch (type) {
1506     case Delete:
1507       return DeleteType.DELETE_ONE_VERSION;
1508     case DeleteColumn:
1509       return DeleteType.DELETE_MULTIPLE_VERSIONS;
1510     case DeleteFamily:
1511       return DeleteType.DELETE_FAMILY;
1512     case DeleteFamilyVersion:
1513       return DeleteType.DELETE_FAMILY_VERSION;
1514     default:
1515         throw new IOException("Unknown delete type: " + type);
1516     }
1517   }
1518 
1519   /**
1520    * Convert a protocol buffer DeleteType to delete KeyValue type.
1521    *
1522    * @param type The DeleteType
1523    * @return The type.
1524    * @throws IOException
1525    */
1526   public static KeyValue.Type fromDeleteType(
1527       DeleteType type) throws IOException {
1528     switch (type) {
1529     case DELETE_ONE_VERSION:
1530       return KeyValue.Type.Delete;
1531     case DELETE_MULTIPLE_VERSIONS:
1532       return KeyValue.Type.DeleteColumn;
1533     case DELETE_FAMILY:
1534       return KeyValue.Type.DeleteFamily;
1535     case DELETE_FAMILY_VERSION:
1536       return KeyValue.Type.DeleteFamilyVersion;
1537     default:
1538       throw new IOException("Unknown delete type: " + type);
1539     }
1540   }
1541 
1542   /**
1543    * Convert a stringified protocol buffer exception Parameter to a Java Exception
1544    *
1545    * @param parameter the protocol buffer Parameter to convert
1546    * @return the converted Exception
1547    * @throws IOException if failed to deserialize the parameter
1548    */
1549   @SuppressWarnings("unchecked")
1550   public static Throwable toException(final NameBytesPair parameter) throws IOException {
1551     if (parameter == null || !parameter.hasValue()) return null;
1552     String desc = parameter.getValue().toStringUtf8();
1553     String type = parameter.getName();
1554     try {
1555       Class<? extends Throwable> c =
1556         (Class<? extends Throwable>)Class.forName(type, true, CLASS_LOADER);
1557       Constructor<? extends Throwable> cn = null;
1558       try {
1559         cn = c.getDeclaredConstructor(String.class);
1560         return cn.newInstance(desc);
1561       } catch (NoSuchMethodException e) {
1562         // Could be a raw RemoteException. See HBASE-8987.
1563         cn = c.getDeclaredConstructor(String.class, String.class);
1564         return cn.newInstance(type, desc);
1565       }
1566     } catch (Exception e) {
1567       throw new IOException(e);
1568     }
1569   }
1570 
1571 // Start helpers for Client
1572 
1573   /**
1574    * A helper to get a row of the closet one before using client protocol.
1575    *
1576    * @param client
1577    * @param regionName
1578    * @param row
1579    * @param family
1580    * @return the row or the closestRowBefore if it doesn't exist
1581    * @throws IOException
1582    * @deprecated since 0.99 - use reversed scanner instead.
1583    */
1584   @Deprecated
1585   public static Result getRowOrBefore(final ClientService.BlockingInterface client,
1586       final byte[] regionName, final byte[] row,
1587       final byte[] family) throws IOException {
1588     GetRequest request =
1589       RequestConverter.buildGetRowOrBeforeRequest(
1590         regionName, row, family);
1591     try {
1592       GetResponse response = client.get(null, request);
1593       if (!response.hasResult()) return null;
1594       return toResult(response.getResult());
1595     } catch (ServiceException se) {
1596       throw getRemoteException(se);
1597     }
1598   }
1599 
1600   /**
1601    * A helper to bulk load a list of HFiles using client protocol.
1602    *
1603    * @param client
1604    * @param familyPaths
1605    * @param regionName
1606    * @param assignSeqNum
1607    * @return true if all are loaded
1608    * @throws IOException
1609    */
1610   public static boolean bulkLoadHFile(final ClientService.BlockingInterface client,
1611       final List<Pair<byte[], String>> familyPaths,
1612       final byte[] regionName, boolean assignSeqNum) throws IOException {
1613     BulkLoadHFileRequest request =
1614       RequestConverter.buildBulkLoadHFileRequest(familyPaths, regionName, assignSeqNum);
1615     try {
1616       BulkLoadHFileResponse response =
1617         client.bulkLoadHFile(null, request);
1618       return response.getLoaded();
1619     } catch (ServiceException se) {
1620       throw getRemoteException(se);
1621     }
1622   }
1623 
1624   public static CoprocessorServiceResponse execService(final RpcController controller,
1625       final ClientService.BlockingInterface client, final CoprocessorServiceCall call,
1626       final byte[] regionName) throws IOException {
1627     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1628         .setCall(call).setRegion(
1629             RequestConverter.buildRegionSpecifier(REGION_NAME, regionName)).build();
1630     try {
1631       CoprocessorServiceResponse response =
1632           client.execService(controller, request);
1633       return response;
1634     } catch (ServiceException se) {
1635       throw getRemoteException(se);
1636     }
1637   }
1638 
1639   public static CoprocessorServiceResponse execService(final RpcController controller,
1640     final MasterService.BlockingInterface client, final CoprocessorServiceCall call)
1641   throws IOException {
1642     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1643         .setCall(call).setRegion(
1644             RequestConverter.buildRegionSpecifier(REGION_NAME, HConstants.EMPTY_BYTE_ARRAY)).build();
1645     try {
1646       CoprocessorServiceResponse response =
1647           client.execMasterService(controller, request);
1648       return response;
1649     } catch (ServiceException se) {
1650       throw getRemoteException(se);
1651     }
1652   }
1653 
1654   /**
1655    * Make a region server endpoint call
1656    * @param client
1657    * @param call
1658    * @return CoprocessorServiceResponse
1659    * @throws IOException
1660    */
1661   public static CoprocessorServiceResponse execRegionServerService(
1662       final RpcController controller, final ClientService.BlockingInterface client,
1663       final CoprocessorServiceCall call)
1664       throws IOException {
1665     CoprocessorServiceRequest request =
1666         CoprocessorServiceRequest
1667             .newBuilder()
1668             .setCall(call)
1669             .setRegion(
1670               RequestConverter.buildRegionSpecifier(REGION_NAME, HConstants.EMPTY_BYTE_ARRAY))
1671             .build();
1672     try {
1673       CoprocessorServiceResponse response = client.execRegionServerService(controller, request);
1674       return response;
1675     } catch (ServiceException se) {
1676       throw getRemoteException(se);
1677     }
1678   }
1679 
1680   @SuppressWarnings("unchecked")
1681   public static <T extends Service> T newServiceStub(Class<T> service, RpcChannel channel)
1682       throws Exception {
1683     return (T)Methods.call(service, null, "newStub",
1684         new Class[]{ RpcChannel.class }, new Object[]{ channel });
1685   }
1686 
1687 // End helpers for Client
1688 // Start helpers for Admin
1689 
1690   /**
1691    * A helper to retrieve region info given a region name
1692    * using admin protocol.
1693    *
1694    * @param admin
1695    * @param regionName
1696    * @return the retrieved region info
1697    * @throws IOException
1698    */
1699   public static HRegionInfo getRegionInfo(final RpcController controller,
1700       final AdminService.BlockingInterface admin, final byte[] regionName) throws IOException {
1701     try {
1702       GetRegionInfoRequest request =
1703         RequestConverter.buildGetRegionInfoRequest(regionName);
1704       GetRegionInfoResponse response =
1705         admin.getRegionInfo(controller, request);
1706       return HRegionInfo.convert(response.getRegionInfo());
1707     } catch (ServiceException se) {
1708       throw getRemoteException(se);
1709     }
1710   }
1711 
1712   /**
1713    * A helper to close a region given a region name
1714    * using admin protocol.
1715    *
1716    * @param admin
1717    * @param regionName
1718    * @param transitionInZK
1719    * @throws IOException
1720    */
1721   public static void closeRegion(final RpcController controller, 
1722       final AdminService.BlockingInterface admin, final ServerName server, final byte[] regionName,
1723       final boolean transitionInZK) throws IOException {
1724     CloseRegionRequest closeRegionRequest =
1725       RequestConverter.buildCloseRegionRequest(server, regionName, transitionInZK);
1726     try {
1727       admin.closeRegion(controller, closeRegionRequest);
1728     } catch (ServiceException se) {
1729       throw getRemoteException(se);
1730     }
1731   }
1732 
1733   /**
1734    * A helper to close a region given a region name
1735    * using admin protocol.
1736    *
1737    * @param admin
1738    * @param regionName
1739    * @param versionOfClosingNode
1740    * @return true if the region is closed
1741    * @throws IOException
1742    */
1743   public static boolean closeRegion(final RpcController controller,
1744       final AdminService.BlockingInterface admin,
1745       final ServerName server,
1746       final byte[] regionName,
1747       final int versionOfClosingNode, final ServerName destinationServer,
1748       final boolean transitionInZK) throws IOException {
1749     CloseRegionRequest closeRegionRequest =
1750       RequestConverter.buildCloseRegionRequest(server,
1751         regionName, versionOfClosingNode, destinationServer, transitionInZK);
1752     try {
1753       CloseRegionResponse response = admin.closeRegion(controller, closeRegionRequest);
1754       return ResponseConverter.isClosed(response);
1755     } catch (ServiceException se) {
1756       throw getRemoteException(se);
1757     }
1758   }
1759 
1760   /**
1761    * A helper to warmup a region given a region name
1762    * using admin protocol
1763    *
1764    * @param admin
1765    * @param regionInfo
1766    *
1767    */
1768   public static void warmupRegion(final RpcController controller,
1769       final AdminService.BlockingInterface admin, final HRegionInfo regionInfo) throws IOException {
1770 
1771     try {
1772       WarmupRegionRequest warmupRegionRequest =
1773            RequestConverter.buildWarmupRegionRequest(regionInfo);
1774 
1775       admin.warmupRegion(controller, warmupRegionRequest);
1776     } catch (ServiceException e) {
1777       throw getRemoteException(e);
1778     }
1779   }
1780 
1781   /**
1782    * A helper to open a region using admin protocol.
1783    * @param admin
1784    * @param region
1785    * @throws IOException
1786    */
1787   public static void openRegion(final RpcController controller,
1788       final AdminService.BlockingInterface admin, ServerName server, final HRegionInfo region)
1789           throws IOException {
1790     OpenRegionRequest request =
1791       RequestConverter.buildOpenRegionRequest(server, region, -1, null, null);
1792     try {
1793       admin.openRegion(controller, request);
1794     } catch (ServiceException se) {
1795       throw ProtobufUtil.getRemoteException(se);
1796     }
1797   }
1798 
1799   /**
1800    * A helper to get the all the online regions on a region
1801    * server using admin protocol.
1802    *
1803    * @param admin
1804    * @return a list of online region info
1805    * @throws IOException
1806    */
1807   public static List<HRegionInfo> getOnlineRegions(final AdminService.BlockingInterface admin)
1808       throws IOException {
1809     return getOnlineRegions(null, admin);
1810   }
1811 
1812   /**
1813    * A helper to get the all the online regions on a region
1814    * server using admin protocol.
1815    * @return a list of online region info
1816    */
1817   public static List<HRegionInfo> getOnlineRegions(final RpcController controller,
1818       final AdminService.BlockingInterface admin)
1819   throws IOException {
1820     GetOnlineRegionRequest request = RequestConverter.buildGetOnlineRegionRequest();
1821     GetOnlineRegionResponse response = null;
1822     try {
1823       response = admin.getOnlineRegion(controller, request);
1824     } catch (ServiceException se) {
1825       throw getRemoteException(se);
1826     }
1827     return getRegionInfos(response);
1828   }
1829 
1830   /**
1831    * Get the list of region info from a GetOnlineRegionResponse
1832    *
1833    * @param proto the GetOnlineRegionResponse
1834    * @return the list of region info or null if <code>proto</code> is null
1835    */
1836   static List<HRegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
1837     if (proto == null) return null;
1838     List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
1839     for (RegionInfo regionInfo: proto.getRegionInfoList()) {
1840       regionInfos.add(HRegionInfo.convert(regionInfo));
1841     }
1842     return regionInfos;
1843   }
1844 
1845   /**
1846    * A helper to get the info of a region server using admin protocol.
1847    * @return the server name
1848    */
1849   public static ServerInfo getServerInfo(final RpcController controller,
1850       final AdminService.BlockingInterface admin)
1851   throws IOException {
1852     GetServerInfoRequest request = RequestConverter.buildGetServerInfoRequest();
1853     try {
1854       GetServerInfoResponse response = admin.getServerInfo(controller, request);
1855       return response.getServerInfo();
1856     } catch (ServiceException se) {
1857       throw getRemoteException(se);
1858     }
1859   }
1860 
1861   /**
1862    * A helper to get the list of files of a column family
1863    * on a given region using admin protocol.
1864    *
1865    * @return the list of store files
1866    */
1867   public static List<String> getStoreFiles(final AdminService.BlockingInterface admin,
1868       final byte[] regionName, final byte[] family)
1869   throws IOException {
1870     return getStoreFiles(null, admin, regionName, family);
1871   }
1872 
1873   /**
1874    * A helper to get the list of files of a column family
1875    * on a given region using admin protocol.
1876    *
1877    * @return the list of store files
1878    */
1879   public static List<String> getStoreFiles(final RpcController controller,
1880       final AdminService.BlockingInterface admin, final byte[] regionName, final byte[] family)
1881   throws IOException {
1882     GetStoreFileRequest request =
1883       RequestConverter.buildGetStoreFileRequest(regionName, family);
1884     try {
1885       GetStoreFileResponse response = admin.getStoreFile(controller, request);
1886       return response.getStoreFileList();
1887     } catch (ServiceException se) {
1888       throw ProtobufUtil.getRemoteException(se);
1889     }
1890   }
1891 
1892   /**
1893    * A helper to split a region using admin protocol.
1894    *
1895    * @param admin
1896    * @param hri
1897    * @param splitPoint
1898    * @throws IOException
1899    */
1900   public static void split(final RpcController controller,
1901       final AdminService.BlockingInterface admin, final HRegionInfo hri, byte[] splitPoint)
1902           throws IOException {
1903     SplitRegionRequest request =
1904       RequestConverter.buildSplitRegionRequest(hri.getRegionName(), splitPoint);
1905     try {
1906       admin.splitRegion(controller, request);
1907     } catch (ServiceException se) {
1908       throw ProtobufUtil.getRemoteException(se);
1909     }
1910   }
1911 
1912   /**
1913    * A helper to merge regions using admin protocol. Send request to
1914    * regionserver.
1915    * @param admin
1916    * @param region_a
1917    * @param region_b
1918    * @param forcible true if do a compulsory merge, otherwise we will only merge
1919    *          two adjacent regions
1920    * @throws IOException
1921    */
1922   public static void mergeRegions(final RpcController controller,
1923       final AdminService.BlockingInterface admin,
1924       final HRegionInfo region_a, final HRegionInfo region_b,
1925       final boolean forcible) throws IOException {
1926     MergeRegionsRequest request = RequestConverter.buildMergeRegionsRequest(
1927         region_a.getRegionName(), region_b.getRegionName(),forcible);
1928     try {
1929       admin.mergeRegions(controller, request);
1930     } catch (ServiceException se) {
1931       throw ProtobufUtil.getRemoteException(se);
1932     }
1933   }
1934 
1935 // End helpers for Admin
1936 
1937   /*
1938    * Get the total (read + write) requests from a RegionLoad pb
1939    * @param rl - RegionLoad pb
1940    * @return total (read + write) requests
1941    */
1942   public static long getTotalRequestsCount(RegionLoad rl) {
1943     if (rl == null) {
1944       return 0;
1945     }
1946 
1947     return rl.getReadRequestsCount() + rl.getWriteRequestsCount();
1948   }
1949 
1950 
1951   /**
1952    * @param m Message to get delimited pb serialization of (with pb magic prefix)
1953    */
1954   public static byte [] toDelimitedByteArray(final Message m) throws IOException {
1955     // Allocate arbitrary big size so we avoid resizing.
1956     ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
1957     baos.write(PB_MAGIC);
1958     m.writeDelimitedTo(baos);
1959     return baos.toByteArray();
1960   }
1961 
1962   /**
1963    * Converts a Permission proto to a client Permission object.
1964    *
1965    * @param proto the protobuf Permission
1966    * @return the converted Permission
1967    */
1968   public static Permission toPermission(AccessControlProtos.Permission proto) {
1969     if (proto.getType() != AccessControlProtos.Permission.Type.Global) {
1970       return toTablePermission(proto);
1971     } else {
1972       List<Permission.Action> actions = toPermissionActions(proto.getGlobalPermission().getActionList());
1973       return new Permission(actions.toArray(new Permission.Action[actions.size()]));
1974     }
1975   }
1976 
1977   /**
1978    * Converts a Permission proto to a client TablePermission object.
1979    *
1980    * @param proto the protobuf Permission
1981    * @return the converted TablePermission
1982    */
1983   public static TablePermission toTablePermission(AccessControlProtos.Permission proto) {
1984     if(proto.getType() == AccessControlProtos.Permission.Type.Global) {
1985       AccessControlProtos.GlobalPermission perm = proto.getGlobalPermission();
1986       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1987 
1988       return new TablePermission(null, null, null,
1989           actions.toArray(new Permission.Action[actions.size()]));
1990     }
1991     if(proto.getType() == AccessControlProtos.Permission.Type.Namespace) {
1992       AccessControlProtos.NamespacePermission perm = proto.getNamespacePermission();
1993       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1994 
1995       if(!proto.hasNamespacePermission()) {
1996         throw new IllegalStateException("Namespace must not be empty in NamespacePermission");
1997       }
1998       String namespace = perm.getNamespaceName().toStringUtf8();
1999       return new TablePermission(namespace, actions.toArray(new Permission.Action[actions.size()]));
2000     }
2001     if(proto.getType() == AccessControlProtos.Permission.Type.Table) {
2002       AccessControlProtos.TablePermission perm = proto.getTablePermission();
2003       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
2004 
2005       byte[] qualifier = null;
2006       byte[] family = null;
2007       TableName table = null;
2008 
2009       if (!perm.hasTableName()) {
2010         throw new IllegalStateException("TableName cannot be empty");
2011       }
2012       table = ProtobufUtil.toTableName(perm.getTableName());
2013 
2014       if (perm.hasFamily()) family = perm.getFamily().toByteArray();
2015       if (perm.hasQualifier()) qualifier = perm.getQualifier().toByteArray();
2016 
2017       return new TablePermission(table, family, qualifier,
2018           actions.toArray(new Permission.Action[actions.size()]));
2019     }
2020     throw new IllegalStateException("Unrecognize Perm Type: "+proto.getType());
2021   }
2022 
2023   /**
2024    * Convert a client Permission to a Permission proto
2025    *
2026    * @param perm the client Permission
2027    * @return the protobuf Permission
2028    */
2029   public static AccessControlProtos.Permission toPermission(Permission perm) {
2030     AccessControlProtos.Permission.Builder ret = AccessControlProtos.Permission.newBuilder();
2031     if (perm instanceof TablePermission) {
2032       TablePermission tablePerm = (TablePermission)perm;
2033       if(tablePerm.hasNamespace()) {
2034         ret.setType(AccessControlProtos.Permission.Type.Namespace);
2035 
2036         AccessControlProtos.NamespacePermission.Builder builder =
2037             AccessControlProtos.NamespacePermission.newBuilder();
2038         builder.setNamespaceName(ByteString.copyFromUtf8(tablePerm.getNamespace()));
2039         Permission.Action actions[] = perm.getActions();
2040         if (actions != null) {
2041           for (Permission.Action a : actions) {
2042             builder.addAction(toPermissionAction(a));
2043           }
2044         }
2045         ret.setNamespacePermission(builder);
2046         return ret.build();
2047       } else if (tablePerm.hasTable()) {
2048         ret.setType(AccessControlProtos.Permission.Type.Table);
2049 
2050         AccessControlProtos.TablePermission.Builder builder =
2051             AccessControlProtos.TablePermission.newBuilder();
2052         builder.setTableName(ProtobufUtil.toProtoTableName(tablePerm.getTableName()));
2053         if (tablePerm.hasFamily()) {
2054           builder.setFamily(ByteStringer.wrap(tablePerm.getFamily()));
2055         }
2056         if (tablePerm.hasQualifier()) {
2057           builder.setQualifier(ByteStringer.wrap(tablePerm.getQualifier()));
2058         }
2059         Permission.Action actions[] = perm.getActions();
2060         if (actions != null) {
2061           for (Permission.Action a : actions) {
2062             builder.addAction(toPermissionAction(a));
2063           }
2064         }
2065         ret.setTablePermission(builder);
2066         return ret.build();
2067       }
2068     }
2069 
2070     ret.setType(AccessControlProtos.Permission.Type.Global);
2071 
2072     AccessControlProtos.GlobalPermission.Builder builder =
2073         AccessControlProtos.GlobalPermission.newBuilder();
2074     Permission.Action actions[] = perm.getActions();
2075     if (actions != null) {
2076       for (Permission.Action a: actions) {
2077         builder.addAction(toPermissionAction(a));
2078       }
2079     }
2080     ret.setGlobalPermission(builder);
2081     return ret.build();
2082   }
2083 
2084   /**
2085    * Converts a list of Permission.Action proto to a list of client Permission.Action objects.
2086    *
2087    * @param protoActions the list of protobuf Actions
2088    * @return the converted list of Actions
2089    */
2090   public static List<Permission.Action> toPermissionActions(
2091       List<AccessControlProtos.Permission.Action> protoActions) {
2092     List<Permission.Action> actions = new ArrayList<Permission.Action>(protoActions.size());
2093     for (AccessControlProtos.Permission.Action a : protoActions) {
2094       actions.add(toPermissionAction(a));
2095     }
2096     return actions;
2097   }
2098 
2099   /**
2100    * Converts a Permission.Action proto to a client Permission.Action object.
2101    *
2102    * @param action the protobuf Action
2103    * @return the converted Action
2104    */
2105   public static Permission.Action toPermissionAction(
2106       AccessControlProtos.Permission.Action action) {
2107     switch (action) {
2108       case READ:
2109         return Permission.Action.READ;
2110       case WRITE:
2111         return Permission.Action.WRITE;
2112       case EXEC:
2113         return Permission.Action.EXEC;
2114       case CREATE:
2115         return Permission.Action.CREATE;
2116       case ADMIN:
2117         return Permission.Action.ADMIN;
2118     }
2119     throw new IllegalArgumentException("Unknown action value "+action.name());
2120   }
2121 
2122   /**
2123    * Convert a client Permission.Action to a Permission.Action proto
2124    *
2125    * @param action the client Action
2126    * @return the protobuf Action
2127    */
2128   public static AccessControlProtos.Permission.Action toPermissionAction(
2129       Permission.Action action) {
2130     switch (action) {
2131       case READ:
2132         return AccessControlProtos.Permission.Action.READ;
2133       case WRITE:
2134         return AccessControlProtos.Permission.Action.WRITE;
2135       case EXEC:
2136         return AccessControlProtos.Permission.Action.EXEC;
2137       case CREATE:
2138         return AccessControlProtos.Permission.Action.CREATE;
2139       case ADMIN:
2140         return AccessControlProtos.Permission.Action.ADMIN;
2141     }
2142     throw new IllegalArgumentException("Unknown action value "+action.name());
2143   }
2144 
2145   /**
2146    * Convert a client user permission to a user permission proto
2147    *
2148    * @param perm the client UserPermission
2149    * @return the protobuf UserPermission
2150    */
2151   public static AccessControlProtos.UserPermission toUserPermission(UserPermission perm) {
2152     return AccessControlProtos.UserPermission.newBuilder()
2153         .setUser(ByteStringer.wrap(perm.getUser()))
2154         .setPermission(toPermission(perm))
2155         .build();
2156   }
2157 
2158   /**
2159    * Converts a user permission proto to a client user permission object.
2160    *
2161    * @param proto the protobuf UserPermission
2162    * @return the converted UserPermission
2163    */
2164   public static UserPermission toUserPermission(AccessControlProtos.UserPermission proto) {
2165     return new UserPermission(proto.getUser().toByteArray(),
2166         toTablePermission(proto.getPermission()));
2167   }
2168 
2169   /**
2170    * Convert a ListMultimap&lt;String, TablePermission&gt; where key is username
2171    * to a protobuf UserPermission
2172    *
2173    * @param perm the list of user and table permissions
2174    * @return the protobuf UserTablePermissions
2175    */
2176   public static AccessControlProtos.UsersAndPermissions toUserTablePermissions(
2177       ListMultimap<String, TablePermission> perm) {
2178     AccessControlProtos.UsersAndPermissions.Builder builder =
2179                   AccessControlProtos.UsersAndPermissions.newBuilder();
2180     for (Map.Entry<String, Collection<TablePermission>> entry : perm.asMap().entrySet()) {
2181       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
2182                   AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
2183       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
2184       for (TablePermission tablePerm: entry.getValue()) {
2185         userPermBuilder.addPermissions(toPermission(tablePerm));
2186       }
2187       builder.addUserPermissions(userPermBuilder.build());
2188     }
2189     return builder.build();
2190   }
2191 
2192   /**
2193    * A utility used to grant a user global permissions.
2194    * <p>
2195    * It's also called by the shell, in case you want to find references.
2196    *
2197    * @param protocol the AccessControlService protocol proxy
2198    * @param userShortName the short name of the user to grant permissions
2199    * @param actions the permissions to be granted
2200    * @throws ServiceException
2201    */
2202   public static void grant(RpcController controller,
2203       AccessControlService.BlockingInterface protocol, String userShortName,
2204       Permission.Action... actions) throws ServiceException {
2205     List<AccessControlProtos.Permission.Action> permActions =
2206         Lists.newArrayListWithCapacity(actions.length);
2207     for (Permission.Action a : actions) {
2208       permActions.add(ProtobufUtil.toPermissionAction(a));
2209     }
2210     AccessControlProtos.GrantRequest request = RequestConverter.
2211       buildGrantRequest(userShortName, permActions.toArray(
2212         new AccessControlProtos.Permission.Action[actions.length]));
2213     protocol.grant(controller, request);
2214   }
2215 
2216   /**
2217    * A utility used to grant a user table permissions. The permissions will
2218    * be for a table table/column family/qualifier.
2219    * <p>
2220    * It's also called by the shell, in case you want to find references.
2221    *
2222    * @param protocol the AccessControlService protocol proxy
2223    * @param userShortName the short name of the user to grant permissions
2224    * @param tableName optional table name
2225    * @param f optional column family
2226    * @param q optional qualifier
2227    * @param actions the permissions to be granted
2228    * @throws ServiceException
2229    */
2230   public static void grant(RpcController controller,
2231       AccessControlService.BlockingInterface protocol, String userShortName, TableName tableName,
2232       byte[] f, byte[] q, Permission.Action... actions) throws ServiceException {
2233     List<AccessControlProtos.Permission.Action> permActions =
2234         Lists.newArrayListWithCapacity(actions.length);
2235     for (Permission.Action a : actions) {
2236       permActions.add(ProtobufUtil.toPermissionAction(a));
2237     }
2238     AccessControlProtos.GrantRequest request = RequestConverter.
2239       buildGrantRequest(userShortName, tableName, f, q, permActions.toArray(
2240         new AccessControlProtos.Permission.Action[actions.length]));
2241     protocol.grant(controller, request);
2242   }
2243 
2244   /**
2245    * A utility used to grant a user namespace permissions.
2246    * <p>
2247    * It's also called by the shell, in case you want to find references.
2248    *
2249    * @param protocol the AccessControlService protocol proxy
2250    * @param namespace the short name of the user to grant permissions
2251    * @param actions the permissions to be granted
2252    * @throws ServiceException
2253    */
2254   public static void grant(RpcController controller,
2255       AccessControlService.BlockingInterface protocol, String userShortName, String namespace,
2256       Permission.Action... actions) throws ServiceException {
2257     List<AccessControlProtos.Permission.Action> permActions =
2258         Lists.newArrayListWithCapacity(actions.length);
2259     for (Permission.Action a : actions) {
2260       permActions.add(ProtobufUtil.toPermissionAction(a));
2261     }
2262     AccessControlProtos.GrantRequest request = RequestConverter.
2263       buildGrantRequest(userShortName, namespace, permActions.toArray(
2264         new AccessControlProtos.Permission.Action[actions.length]));
2265     protocol.grant(controller, request);
2266   }
2267 
2268   /**
2269    * A utility used to revoke a user's global permissions.
2270    * <p>
2271    * It's also called by the shell, in case you want to find references.
2272    *
2273    * @param protocol the AccessControlService protocol proxy
2274    * @param userShortName the short name of the user to revoke permissions
2275    * @param actions the permissions to be revoked
2276    * @throws ServiceException
2277    */
2278   public static void revoke(RpcController controller,
2279       AccessControlService.BlockingInterface protocol, String userShortName,
2280       Permission.Action... actions) throws ServiceException {
2281     List<AccessControlProtos.Permission.Action> permActions =
2282         Lists.newArrayListWithCapacity(actions.length);
2283     for (Permission.Action a : actions) {
2284       permActions.add(ProtobufUtil.toPermissionAction(a));
2285     }
2286     AccessControlProtos.RevokeRequest request = RequestConverter.
2287       buildRevokeRequest(userShortName, permActions.toArray(
2288         new AccessControlProtos.Permission.Action[actions.length]));
2289     protocol.revoke(controller, request);
2290   }
2291 
2292   /**
2293    * A utility used to revoke a user's table permissions. The permissions will
2294    * be for a table/column family/qualifier.
2295    * <p>
2296    * It's also called by the shell, in case you want to find references.
2297    *
2298    * @param protocol the AccessControlService protocol proxy
2299    * @param userShortName the short name of the user to revoke permissions
2300    * @param tableName optional table name
2301    * @param f optional column family
2302    * @param q optional qualifier
2303    * @param actions the permissions to be revoked
2304    * @throws ServiceException
2305    */
2306   public static void revoke(RpcController controller,
2307       AccessControlService.BlockingInterface protocol, String userShortName, TableName tableName,
2308       byte[] f, byte[] q, Permission.Action... actions) throws ServiceException {
2309     List<AccessControlProtos.Permission.Action> permActions =
2310         Lists.newArrayListWithCapacity(actions.length);
2311     for (Permission.Action a : actions) {
2312       permActions.add(ProtobufUtil.toPermissionAction(a));
2313     }
2314     AccessControlProtos.RevokeRequest request = RequestConverter.
2315       buildRevokeRequest(userShortName, tableName, f, q, permActions.toArray(
2316         new AccessControlProtos.Permission.Action[actions.length]));
2317     protocol.revoke(controller, request);
2318   }
2319 
2320   /**
2321    * A utility used to revoke a user's namespace permissions.
2322    * <p>
2323    * It's also called by the shell, in case you want to find references.
2324    *
2325    * @param protocol the AccessControlService protocol proxy
2326    * @param userShortName the short name of the user to revoke permissions
2327    * @param namespace optional table name
2328    * @param actions the permissions to be revoked
2329    * @throws ServiceException
2330    */
2331   public static void revoke(RpcController controller,
2332       AccessControlService.BlockingInterface protocol, String userShortName, String namespace,
2333       Permission.Action... actions) throws ServiceException {
2334     List<AccessControlProtos.Permission.Action> permActions =
2335         Lists.newArrayListWithCapacity(actions.length);
2336     for (Permission.Action a : actions) {
2337       permActions.add(ProtobufUtil.toPermissionAction(a));
2338     }
2339     AccessControlProtos.RevokeRequest request = RequestConverter.
2340       buildRevokeRequest(userShortName, namespace, permActions.toArray(
2341         new AccessControlProtos.Permission.Action[actions.length]));
2342     protocol.revoke(controller, request);
2343   }
2344 
2345   /**
2346    * A utility used to get user's global permissions.
2347    * <p>
2348    * It's also called by the shell, in case you want to find references.
2349    *
2350    * @param protocol the AccessControlService protocol proxy
2351    * @throws ServiceException
2352    */
2353   public static List<UserPermission> getUserPermissions(RpcController controller,
2354       AccessControlService.BlockingInterface protocol) throws ServiceException {
2355     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2356       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2357     builder.setType(AccessControlProtos.Permission.Type.Global);
2358     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2359     AccessControlProtos.GetUserPermissionsResponse response =
2360       protocol.getUserPermissions(controller, request);
2361     List<UserPermission> perms = new ArrayList<UserPermission>(response.getUserPermissionCount());
2362     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2363       perms.add(ProtobufUtil.toUserPermission(perm));
2364     }
2365     return perms;
2366   }
2367 
2368   /**
2369    * A utility used to get user table permissions.
2370    * <p>
2371    * It's also called by the shell, in case you want to find references.
2372    *
2373    * @param protocol the AccessControlService protocol proxy
2374    * @param t optional table name
2375    * @throws ServiceException
2376    */
2377   public static List<UserPermission> getUserPermissions(RpcController controller,
2378       AccessControlService.BlockingInterface protocol,
2379       TableName t) throws ServiceException {
2380     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2381       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2382     if (t != null) {
2383       builder.setTableName(ProtobufUtil.toProtoTableName(t));
2384     }
2385     builder.setType(AccessControlProtos.Permission.Type.Table);
2386     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2387     AccessControlProtos.GetUserPermissionsResponse response =
2388       protocol.getUserPermissions(controller, request);
2389     List<UserPermission> perms = new ArrayList<UserPermission>(response.getUserPermissionCount());
2390     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2391       perms.add(ProtobufUtil.toUserPermission(perm));
2392     }
2393     return perms;
2394   }
2395 
2396   /**
2397    * A utility used to get permissions for selected namespace.
2398    * <p>
2399    * It's also called by the shell, in case you want to find references.
2400    *
2401    * @param protocol the AccessControlService protocol proxy
2402    * @param namespace name of the namespace
2403    * @throws ServiceException
2404    */
2405   public static List<UserPermission> getUserPermissions(RpcController controller,
2406       AccessControlService.BlockingInterface protocol,
2407       byte[] namespace) throws ServiceException {
2408     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2409       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2410     if (namespace != null) {
2411       builder.setNamespaceName(ByteStringer.wrap(namespace));
2412     }
2413     builder.setType(AccessControlProtos.Permission.Type.Namespace);
2414     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2415     AccessControlProtos.GetUserPermissionsResponse response =
2416       protocol.getUserPermissions(controller, request);
2417     List<UserPermission> perms = new ArrayList<UserPermission>(response.getUserPermissionCount());
2418     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2419       perms.add(ProtobufUtil.toUserPermission(perm));
2420     }
2421     return perms;
2422   }
2423 
2424   /**
2425    * Convert a protobuf UserTablePermissions to a
2426    * ListMultimap&lt;String, TablePermission&gt; where key is username.
2427    *
2428    * @param proto the protobuf UserPermission
2429    * @return the converted UserPermission
2430    */
2431   public static ListMultimap<String, TablePermission> toUserTablePermissions(
2432       AccessControlProtos.UsersAndPermissions proto) {
2433     ListMultimap<String, TablePermission> perms = ArrayListMultimap.create();
2434     AccessControlProtos.UsersAndPermissions.UserPermissions userPerm;
2435 
2436     for (int i = 0; i < proto.getUserPermissionsCount(); i++) {
2437       userPerm = proto.getUserPermissions(i);
2438       for (int j = 0; j < userPerm.getPermissionsCount(); j++) {
2439         TablePermission tablePerm = toTablePermission(userPerm.getPermissions(j));
2440         perms.put(userPerm.getUser().toStringUtf8(), tablePerm);
2441       }
2442     }
2443 
2444     return perms;
2445   }
2446 
2447   /**
2448    * Converts a Token instance (with embedded identifier) to the protobuf representation.
2449    *
2450    * @param token the Token instance to copy
2451    * @return the protobuf Token message
2452    */
2453   public static AuthenticationProtos.Token toToken(Token<AuthenticationTokenIdentifier> token) {
2454     AuthenticationProtos.Token.Builder builder = AuthenticationProtos.Token.newBuilder();
2455     builder.setIdentifier(ByteStringer.wrap(token.getIdentifier()));
2456     builder.setPassword(ByteStringer.wrap(token.getPassword()));
2457     if (token.getService() != null) {
2458       builder.setService(ByteString.copyFromUtf8(token.getService().toString()));
2459     }
2460     return builder.build();
2461   }
2462 
2463   /**
2464    * Converts a protobuf Token message back into a Token instance.
2465    *
2466    * @param proto the protobuf Token message
2467    * @return the Token instance
2468    */
2469   public static Token<AuthenticationTokenIdentifier> toToken(AuthenticationProtos.Token proto) {
2470     return new Token<AuthenticationTokenIdentifier>(
2471         proto.hasIdentifier() ? proto.getIdentifier().toByteArray() : null,
2472         proto.hasPassword() ? proto.getPassword().toByteArray() : null,
2473         AuthenticationTokenIdentifier.AUTH_TOKEN_TYPE,
2474         proto.hasService() ? new Text(proto.getService().toStringUtf8()) : null);
2475   }
2476 
2477   /**
2478    * Find the HRegion encoded name based on a region specifier
2479    *
2480    * @param regionSpecifier the region specifier
2481    * @return the corresponding region's encoded name
2482    * @throws DoNotRetryIOException if the specifier type is unsupported
2483    */
2484   public static String getRegionEncodedName(
2485       final RegionSpecifier regionSpecifier) throws DoNotRetryIOException {
2486     byte[] value = regionSpecifier.getValue().toByteArray();
2487     RegionSpecifierType type = regionSpecifier.getType();
2488     switch (type) {
2489       case REGION_NAME:
2490         return HRegionInfo.encodeRegionName(value);
2491       case ENCODED_REGION_NAME:
2492         return Bytes.toString(value);
2493       default:
2494         throw new DoNotRetryIOException(
2495           "Unsupported region specifier type: " + type);
2496     }
2497   }
2498 
2499   public static ScanMetrics toScanMetrics(final byte[] bytes) {
2500     Parser<MapReduceProtos.ScanMetrics> parser = MapReduceProtos.ScanMetrics.PARSER;
2501     MapReduceProtos.ScanMetrics pScanMetrics = null;
2502     try {
2503       pScanMetrics = parser.parseFrom(bytes);
2504     } catch (InvalidProtocolBufferException e) {
2505       //Ignored there are just no key values to add.
2506     }
2507     ScanMetrics scanMetrics = new ScanMetrics();
2508     if (pScanMetrics != null) {
2509       for (HBaseProtos.NameInt64Pair pair : pScanMetrics.getMetricsList()) {
2510         if (pair.hasName() && pair.hasValue()) {
2511           scanMetrics.setCounter(pair.getName(), pair.getValue());
2512         }
2513       }
2514     }
2515     return scanMetrics;
2516   }
2517 
2518   public static MapReduceProtos.ScanMetrics toScanMetrics(ScanMetrics scanMetrics) {
2519     MapReduceProtos.ScanMetrics.Builder builder = MapReduceProtos.ScanMetrics.newBuilder();
2520     Map<String, Long> metrics = scanMetrics.getMetricsMap();
2521     for (Entry<String, Long> e : metrics.entrySet()) {
2522       HBaseProtos.NameInt64Pair nameInt64Pair =
2523           HBaseProtos.NameInt64Pair.newBuilder()
2524               .setName(e.getKey())
2525               .setValue(e.getValue())
2526               .build();
2527       builder.addMetrics(nameInt64Pair);
2528     }
2529     return builder.build();
2530   }
2531 
2532   /**
2533    * Unwraps an exception from a protobuf service into the underlying (expected) IOException.
2534    * This method will <strong>always</strong> throw an exception.
2535    * @param se the {@code ServiceException} instance to convert into an {@code IOException}
2536    */
2537   public static void toIOException(ServiceException se) throws IOException {
2538     if (se == null) {
2539       throw new NullPointerException("Null service exception passed!");
2540     }
2541 
2542     Throwable cause = se.getCause();
2543     if (cause != null && cause instanceof IOException) {
2544       throw (IOException)cause;
2545     }
2546     throw new IOException(se);
2547   }
2548 
2549   public static CellProtos.Cell toCell(final Cell kv) {
2550     // Doing this is going to kill us if we do it for all data passed.
2551     // St.Ack 20121205
2552     CellProtos.Cell.Builder kvbuilder = CellProtos.Cell.newBuilder();
2553     kvbuilder.setRow(ByteStringer.wrap(kv.getRowArray(), kv.getRowOffset(),
2554         kv.getRowLength()));
2555     kvbuilder.setFamily(ByteStringer.wrap(kv.getFamilyArray(),
2556         kv.getFamilyOffset(), kv.getFamilyLength()));
2557     kvbuilder.setQualifier(ByteStringer.wrap(kv.getQualifierArray(),
2558         kv.getQualifierOffset(), kv.getQualifierLength()));
2559     kvbuilder.setCellType(CellProtos.CellType.valueOf(kv.getTypeByte()));
2560     kvbuilder.setTimestamp(kv.getTimestamp());
2561     kvbuilder.setValue(ByteStringer.wrap(kv.getValueArray(), kv.getValueOffset(),
2562         kv.getValueLength()));
2563     return kvbuilder.build();
2564   }
2565 
2566   public static Cell toCell(final CellProtos.Cell cell) {
2567     // Doing this is going to kill us if we do it for all data passed.
2568     // St.Ack 20121205
2569     return CellUtil.createCell(cell.getRow().toByteArray(),
2570       cell.getFamily().toByteArray(),
2571       cell.getQualifier().toByteArray(),
2572       cell.getTimestamp(),
2573       (byte)cell.getCellType().getNumber(),
2574       cell.getValue().toByteArray());
2575   }
2576 
2577   public static HBaseProtos.NamespaceDescriptor toProtoNamespaceDescriptor(NamespaceDescriptor ns) {
2578     HBaseProtos.NamespaceDescriptor.Builder b =
2579         HBaseProtos.NamespaceDescriptor.newBuilder()
2580             .setName(ByteString.copyFromUtf8(ns.getName()));
2581     for(Map.Entry<String, String> entry: ns.getConfiguration().entrySet()) {
2582       b.addConfiguration(HBaseProtos.NameStringPair.newBuilder()
2583           .setName(entry.getKey())
2584           .setValue(entry.getValue()));
2585     }
2586     return b.build();
2587   }
2588 
2589   public static NamespaceDescriptor toNamespaceDescriptor(
2590       HBaseProtos.NamespaceDescriptor desc) throws IOException {
2591     NamespaceDescriptor.Builder b =
2592       NamespaceDescriptor.create(desc.getName().toStringUtf8());
2593     for(HBaseProtos.NameStringPair prop : desc.getConfigurationList()) {
2594       b.addConfiguration(prop.getName(), prop.getValue());
2595     }
2596     return b.build();
2597   }
2598 
2599   /**
2600    * Get an instance of the argument type declared in a class's signature. The
2601    * argument type is assumed to be a PB Message subclass, and the instance is
2602    * created using parseFrom method on the passed ByteString.
2603    * @param runtimeClass the runtime type of the class
2604    * @param position the position of the argument in the class declaration
2605    * @param b the ByteString which should be parsed to get the instance created
2606    * @return the instance
2607    * @throws IOException
2608    */
2609   @SuppressWarnings("unchecked")
2610   public static <T extends Message>
2611   T getParsedGenericInstance(Class<?> runtimeClass, int position, ByteString b)
2612       throws IOException {
2613     Type type = runtimeClass.getGenericSuperclass();
2614     Type argType = ((ParameterizedType)type).getActualTypeArguments()[position];
2615     Class<T> classType = (Class<T>)argType;
2616     T inst;
2617     try {
2618       Method m = classType.getMethod("parseFrom", ByteString.class);
2619       inst = (T)m.invoke(null, b);
2620       return inst;
2621     } catch (SecurityException e) {
2622       throw new IOException(e);
2623     } catch (NoSuchMethodException e) {
2624       throw new IOException(e);
2625     } catch (IllegalArgumentException e) {
2626       throw new IOException(e);
2627     } catch (InvocationTargetException e) {
2628       throw new IOException(e);
2629     } catch (IllegalAccessException e) {
2630       throw new IOException(e);
2631     }
2632   }
2633 
2634   public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family,
2635       List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
2636     return toCompactionDescriptor(info, null, family, inputPaths, outputPaths, storeDir);
2637   }
2638 
2639   @SuppressWarnings("deprecation")
2640   public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] regionName,
2641       byte[] family, List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
2642     // compaction descriptor contains relative paths.
2643     // input / output paths are relative to the store dir
2644     // store dir is relative to region dir
2645     CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder()
2646         .setTableName(ByteStringer.wrap(info.getTableName()))
2647         .setEncodedRegionName(ByteStringer.wrap(
2648           regionName == null ? info.getEncodedNameAsBytes() : regionName))
2649         .setFamilyName(ByteStringer.wrap(family))
2650         .setStoreHomeDir(storeDir.getName()); //make relative
2651     for (Path inputPath : inputPaths) {
2652       builder.addCompactionInput(inputPath.getName()); //relative path
2653     }
2654     for (Path outputPath : outputPaths) {
2655       builder.addCompactionOutput(outputPath.getName());
2656     }
2657     builder.setRegionName(ByteStringer.wrap(info.getRegionName()));
2658     return builder.build();
2659   }
2660 
2661   public static FlushDescriptor toFlushDescriptor(FlushAction action, HRegionInfo hri,
2662       long flushSeqId, Map<byte[], List<Path>> committedFiles) {
2663     FlushDescriptor.Builder desc = FlushDescriptor.newBuilder()
2664         .setAction(action)
2665         .setEncodedRegionName(ByteStringer.wrap(hri.getEncodedNameAsBytes()))
2666         .setRegionName(ByteStringer.wrap(hri.getRegionName()))
2667         .setFlushSequenceNumber(flushSeqId)
2668         .setTableName(ByteStringer.wrap(hri.getTable().getName()));
2669 
2670     for (Map.Entry<byte[], List<Path>> entry : committedFiles.entrySet()) {
2671       WALProtos.FlushDescriptor.StoreFlushDescriptor.Builder builder =
2672           WALProtos.FlushDescriptor.StoreFlushDescriptor.newBuilder()
2673           .setFamilyName(ByteStringer.wrap(entry.getKey()))
2674           .setStoreHomeDir(Bytes.toString(entry.getKey())); //relative to region
2675       if (entry.getValue() != null) {
2676         for (Path path : entry.getValue()) {
2677           builder.addFlushOutput(path.getName());
2678         }
2679       }
2680       desc.addStoreFlushes(builder);
2681     }
2682     return desc.build();
2683   }
2684 
2685   public static RegionEventDescriptor toRegionEventDescriptor(
2686       EventType eventType, HRegionInfo hri, long seqId, ServerName server,
2687       Map<byte[], List<Path>> storeFiles) {
2688     final byte[] tableNameAsBytes = hri.getTable().getName();
2689     final byte[] encodedNameAsBytes = hri.getEncodedNameAsBytes();
2690     final byte[] regionNameAsBytes = hri.getRegionName();
2691     return toRegionEventDescriptor(eventType,
2692         tableNameAsBytes,
2693         encodedNameAsBytes,
2694         regionNameAsBytes,
2695         seqId,
2696 
2697         server,
2698         storeFiles);
2699   }
2700 
2701   public static RegionEventDescriptor toRegionEventDescriptor(EventType eventType,
2702                                                               byte[] tableNameAsBytes,
2703                                                               byte[] encodedNameAsBytes,
2704                                                               byte[] regionNameAsBytes,
2705                                                                long seqId,
2706 
2707                                                               ServerName server,
2708                                                               Map<byte[], List<Path>> storeFiles) {
2709     RegionEventDescriptor.Builder desc = RegionEventDescriptor.newBuilder()
2710         .setEventType(eventType)
2711         .setTableName(ByteStringer.wrap(tableNameAsBytes))
2712         .setEncodedRegionName(ByteStringer.wrap(encodedNameAsBytes))
2713         .setRegionName(ByteStringer.wrap(regionNameAsBytes))
2714         .setLogSequenceNumber(seqId)
2715         .setServer(toServerName(server));
2716 
2717     for (Entry<byte[], List<Path>> entry : storeFiles.entrySet()) {
2718       StoreDescriptor.Builder builder = StoreDescriptor.newBuilder()
2719           .setFamilyName(ByteStringer.wrap(entry.getKey()))
2720           .setStoreHomeDir(Bytes.toString(entry.getKey()));
2721       for (Path path : entry.getValue()) {
2722         builder.addStoreFile(path.getName());
2723       }
2724 
2725       desc.addStores(builder);
2726     }
2727     return desc.build();
2728   }
2729 
2730   /**
2731    * Return short version of Message toString'd, shorter than TextFormat#shortDebugString.
2732    * Tries to NOT print out data both because it can be big but also so we do not have data in our
2733    * logs. Use judiciously.
2734    * @param m
2735    * @return toString of passed <code>m</code>
2736    */
2737   public static String getShortTextFormat(Message m) {
2738     if (m == null) return "null";
2739     if (m instanceof ScanRequest) {
2740       // This should be small and safe to output.  No data.
2741       return TextFormat.shortDebugString(m);
2742     } else if (m instanceof RegionServerReportRequest) {
2743       // Print a short message only, just the servername and the requests, not the full load.
2744       RegionServerReportRequest r = (RegionServerReportRequest)m;
2745       return "server " + TextFormat.shortDebugString(r.getServer()) +
2746         " load { numberOfRequests: " + r.getLoad().getNumberOfRequests() + " }";
2747     } else if (m instanceof RegionServerStartupRequest) {
2748       // Should be small enough.
2749       return TextFormat.shortDebugString(m);
2750     } else if (m instanceof MutationProto) {
2751       return toShortString((MutationProto)m);
2752     } else if (m instanceof GetRequest) {
2753       GetRequest r = (GetRequest) m;
2754       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2755           ", row=" + getStringForByteString(r.getGet().getRow());
2756     } else if (m instanceof ClientProtos.MultiRequest) {
2757       ClientProtos.MultiRequest r = (ClientProtos.MultiRequest) m;
2758       // Get first set of Actions.
2759       ClientProtos.RegionAction actions = r.getRegionActionList().get(0);
2760       String row = actions.getActionCount() <= 0? "":
2761         getStringForByteString(actions.getAction(0).hasGet()?
2762           actions.getAction(0).getGet().getRow():
2763           actions.getAction(0).getMutation().getRow());
2764       return "region= " + getStringForByteString(actions.getRegion().getValue()) +
2765           ", for " + r.getRegionActionCount() +
2766           " actions and 1st row key=" + row;
2767     } else if (m instanceof ClientProtos.MutateRequest) {
2768       ClientProtos.MutateRequest r = (ClientProtos.MutateRequest) m;
2769       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2770           ", row=" + getStringForByteString(r.getMutation().getRow());
2771     }
2772     return "TODO: " + m.getClass().toString();
2773   }
2774 
2775   private static String getStringForByteString(ByteString bs) {
2776     return Bytes.toStringBinary(bs.toByteArray());
2777   }
2778 
2779   /**
2780    * Print out some subset of a MutationProto rather than all of it and its data
2781    * @param proto Protobuf to print out
2782    * @return Short String of mutation proto
2783    */
2784   static String toShortString(final MutationProto proto) {
2785     return "row=" + Bytes.toString(proto.getRow().toByteArray()) +
2786         ", type=" + proto.getMutateType().toString();
2787   }
2788 
2789   public static TableName toTableName(HBaseProtos.TableName tableNamePB) {
2790     return TableName.valueOf(tableNamePB.getNamespace().asReadOnlyByteBuffer(),
2791         tableNamePB.getQualifier().asReadOnlyByteBuffer());
2792   }
2793 
2794   public static HBaseProtos.TableName toProtoTableName(TableName tableName) {
2795     return HBaseProtos.TableName.newBuilder()
2796         .setNamespace(ByteStringer.wrap(tableName.getNamespace()))
2797         .setQualifier(ByteStringer.wrap(tableName.getQualifier())).build();
2798   }
2799 
2800   public static TableName[] getTableNameArray(List<HBaseProtos.TableName> tableNamesList) {
2801     if (tableNamesList == null) {
2802       return new TableName[0];
2803     }
2804     TableName[] tableNames = new TableName[tableNamesList.size()];
2805     for (int i = 0; i < tableNamesList.size(); i++) {
2806       tableNames[i] = toTableName(tableNamesList.get(i));
2807     }
2808     return tableNames;
2809   }
2810 
2811   /**
2812    * Convert a protocol buffer CellVisibility to a client CellVisibility
2813    *
2814    * @param proto
2815    * @return the converted client CellVisibility
2816    */
2817   public static CellVisibility toCellVisibility(ClientProtos.CellVisibility proto) {
2818     if (proto == null) return null;
2819     return new CellVisibility(proto.getExpression());
2820   }
2821 
2822   /**
2823    * Convert a protocol buffer CellVisibility bytes to a client CellVisibility
2824    *
2825    * @param protoBytes
2826    * @return the converted client CellVisibility
2827    * @throws DeserializationException
2828    */
2829   public static CellVisibility toCellVisibility(byte[] protoBytes) throws DeserializationException {
2830     if (protoBytes == null) return null;
2831     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2832     ClientProtos.CellVisibility proto = null;
2833     try {
2834       ProtobufUtil.mergeFrom(builder, protoBytes);
2835       proto = builder.build();
2836     } catch (IOException e) {
2837       throw new DeserializationException(e);
2838     }
2839     return toCellVisibility(proto);
2840   }
2841 
2842   /**
2843    * Create a protocol buffer CellVisibility based on a client CellVisibility.
2844    *
2845    * @param cellVisibility
2846    * @return a protocol buffer CellVisibility
2847    */
2848   public static ClientProtos.CellVisibility toCellVisibility(CellVisibility cellVisibility) {
2849     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2850     builder.setExpression(cellVisibility.getExpression());
2851     return builder.build();
2852   }
2853 
2854   /**
2855    * Convert a protocol buffer Authorizations to a client Authorizations
2856    *
2857    * @param proto
2858    * @return the converted client Authorizations
2859    */
2860   public static Authorizations toAuthorizations(ClientProtos.Authorizations proto) {
2861     if (proto == null) return null;
2862     return new Authorizations(proto.getLabelList());
2863   }
2864 
2865   /**
2866    * Convert a protocol buffer Authorizations bytes to a client Authorizations
2867    *
2868    * @param protoBytes
2869    * @return the converted client Authorizations
2870    * @throws DeserializationException
2871    */
2872   public static Authorizations toAuthorizations(byte[] protoBytes) throws DeserializationException {
2873     if (protoBytes == null) return null;
2874     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2875     ClientProtos.Authorizations proto = null;
2876     try {
2877       ProtobufUtil.mergeFrom(builder, protoBytes);
2878       proto = builder.build();
2879     } catch (IOException e) {
2880       throw new DeserializationException(e);
2881     }
2882     return toAuthorizations(proto);
2883   }
2884 
2885   /**
2886    * Create a protocol buffer Authorizations based on a client Authorizations.
2887    *
2888    * @param authorizations
2889    * @return a protocol buffer Authorizations
2890    */
2891   public static ClientProtos.Authorizations toAuthorizations(Authorizations authorizations) {
2892     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2893     for (String label : authorizations.getLabels()) {
2894       builder.addLabel(label);
2895     }
2896     return builder.build();
2897   }
2898 
2899   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(String user,
2900       Permission perms) {
2901     return AccessControlProtos.UsersAndPermissions.newBuilder()
2902       .addUserPermissions(AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder()
2903         .setUser(ByteString.copyFromUtf8(user))
2904         .addPermissions(toPermission(perms))
2905         .build())
2906       .build();
2907   }
2908 
2909   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(
2910       ListMultimap<String, Permission> perms) {
2911     AccessControlProtos.UsersAndPermissions.Builder builder =
2912         AccessControlProtos.UsersAndPermissions.newBuilder();
2913     for (Map.Entry<String, Collection<Permission>> entry : perms.asMap().entrySet()) {
2914       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
2915         AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
2916       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
2917       for (Permission perm: entry.getValue()) {
2918         userPermBuilder.addPermissions(toPermission(perm));
2919       }
2920       builder.addUserPermissions(userPermBuilder.build());
2921     }
2922     return builder.build();
2923   }
2924 
2925   public static ListMultimap<String, Permission> toUsersAndPermissions(
2926       AccessControlProtos.UsersAndPermissions proto) {
2927     ListMultimap<String, Permission> result = ArrayListMultimap.create();
2928     for (AccessControlProtos.UsersAndPermissions.UserPermissions userPerms:
2929         proto.getUserPermissionsList()) {
2930       String user = userPerms.getUser().toStringUtf8();
2931       for (AccessControlProtos.Permission perm: userPerms.getPermissionsList()) {
2932         result.put(user, toPermission(perm));
2933       }
2934     }
2935     return result;
2936   }
2937   
2938   /**
2939    * Convert a protocol buffer TimeUnit to a client TimeUnit
2940    * @param proto
2941    * @return the converted client TimeUnit
2942    */
2943   public static TimeUnit toTimeUnit(final HBaseProtos.TimeUnit proto) {
2944     switch (proto) {
2945     case NANOSECONDS:
2946       return TimeUnit.NANOSECONDS;
2947     case MICROSECONDS:
2948       return TimeUnit.MICROSECONDS;
2949     case MILLISECONDS:
2950       return TimeUnit.MILLISECONDS;
2951     case SECONDS:
2952       return TimeUnit.SECONDS;
2953     case MINUTES:
2954       return TimeUnit.MINUTES;
2955     case HOURS:
2956       return TimeUnit.HOURS;
2957     case DAYS:
2958       return TimeUnit.DAYS;
2959     default:
2960       throw new RuntimeException("Invalid TimeUnit " + proto);
2961     }
2962   }
2963 
2964   /**
2965    * Convert a client TimeUnit to a protocol buffer TimeUnit
2966    * @param timeUnit
2967    * @return the converted protocol buffer TimeUnit
2968    */
2969   public static HBaseProtos.TimeUnit toProtoTimeUnit(final TimeUnit timeUnit) {
2970     switch (timeUnit) {
2971     case NANOSECONDS:
2972       return HBaseProtos.TimeUnit.NANOSECONDS;
2973     case MICROSECONDS:
2974       return HBaseProtos.TimeUnit.MICROSECONDS;
2975     case MILLISECONDS:
2976       return HBaseProtos.TimeUnit.MILLISECONDS;
2977     case SECONDS:
2978       return HBaseProtos.TimeUnit.SECONDS;
2979     case MINUTES:
2980       return HBaseProtos.TimeUnit.MINUTES;
2981     case HOURS:
2982       return HBaseProtos.TimeUnit.HOURS;
2983     case DAYS:
2984       return HBaseProtos.TimeUnit.DAYS;
2985     default:
2986       throw new RuntimeException("Invalid TimeUnit " + timeUnit);
2987     }
2988   }
2989 
2990   /**
2991    * Convert a protocol buffer ThrottleType to a client ThrottleType
2992    * @param proto
2993    * @return the converted client ThrottleType
2994    */
2995   public static ThrottleType toThrottleType(final QuotaProtos.ThrottleType proto) {
2996     switch (proto) {
2997     case REQUEST_NUMBER:
2998       return ThrottleType.REQUEST_NUMBER;
2999     case REQUEST_SIZE:
3000       return ThrottleType.REQUEST_SIZE;
3001     case WRITE_NUMBER:
3002       return ThrottleType.WRITE_NUMBER;
3003     case WRITE_SIZE:
3004       return ThrottleType.WRITE_SIZE;
3005     case READ_NUMBER:
3006       return ThrottleType.READ_NUMBER;
3007     case READ_SIZE:
3008       return ThrottleType.READ_SIZE;
3009     default:
3010       throw new RuntimeException("Invalid ThrottleType " + proto);
3011     }
3012   }
3013 
3014   /**
3015    * Convert a client ThrottleType to a protocol buffer ThrottleType
3016    * @param type
3017    * @return the converted protocol buffer ThrottleType
3018    */
3019   public static QuotaProtos.ThrottleType toProtoThrottleType(final ThrottleType type) {
3020     switch (type) {
3021     case REQUEST_NUMBER:
3022       return QuotaProtos.ThrottleType.REQUEST_NUMBER;
3023     case REQUEST_SIZE:
3024       return QuotaProtos.ThrottleType.REQUEST_SIZE;
3025     case WRITE_NUMBER:
3026       return QuotaProtos.ThrottleType.WRITE_NUMBER;
3027     case WRITE_SIZE:
3028       return QuotaProtos.ThrottleType.WRITE_SIZE;
3029     case READ_NUMBER:
3030       return QuotaProtos.ThrottleType.READ_NUMBER;
3031     case READ_SIZE:
3032       return QuotaProtos.ThrottleType.READ_SIZE;
3033     default:
3034       throw new RuntimeException("Invalid ThrottleType " + type);
3035     }
3036   }
3037 
3038   /**
3039    * Convert a protocol buffer QuotaScope to a client QuotaScope
3040    * @param proto
3041    * @return the converted client QuotaScope
3042    */
3043   public static QuotaScope toQuotaScope(final QuotaProtos.QuotaScope proto) {
3044     switch (proto) {
3045     case CLUSTER:
3046       return QuotaScope.CLUSTER;
3047     case MACHINE:
3048       return QuotaScope.MACHINE;
3049     default:
3050       throw new RuntimeException("Invalid QuotaScope " + proto);
3051     }
3052   }
3053 
3054   /**
3055    * Convert a client QuotaScope to a protocol buffer QuotaScope
3056    * @param scope
3057    * @return the converted protocol buffer QuotaScope
3058    */
3059   public static QuotaProtos.QuotaScope toProtoQuotaScope(final QuotaScope scope) {
3060     switch (scope) {
3061     case CLUSTER:
3062       return QuotaProtos.QuotaScope.CLUSTER;
3063     case MACHINE:
3064       return QuotaProtos.QuotaScope.MACHINE;
3065     default:
3066       throw new RuntimeException("Invalid QuotaScope " + scope);
3067     }
3068   }
3069 
3070   /**
3071    * Convert a protocol buffer QuotaType to a client QuotaType
3072    * @param proto
3073    * @return the converted client QuotaType
3074    */
3075   public static QuotaType toQuotaScope(final QuotaProtos.QuotaType proto) {
3076     switch (proto) {
3077     case THROTTLE:
3078       return QuotaType.THROTTLE;
3079     default:
3080       throw new RuntimeException("Invalid QuotaType " + proto);
3081     }
3082   }
3083 
3084   /**
3085    * Convert a client QuotaType to a protocol buffer QuotaType
3086    * @param type
3087    * @return the converted protocol buffer QuotaType
3088    */
3089   public static QuotaProtos.QuotaType toProtoQuotaScope(final QuotaType type) {
3090     switch (type) {
3091     case THROTTLE:
3092       return QuotaProtos.QuotaType.THROTTLE;
3093     default:
3094       throw new RuntimeException("Invalid QuotaType " + type);
3095     }
3096   }
3097 
3098   /**
3099    * Build a protocol buffer TimedQuota
3100    * @param limit the allowed number of request/data per timeUnit
3101    * @param timeUnit the limit time unit
3102    * @param scope the quota scope
3103    * @return the protocol buffer TimedQuota
3104    */
3105   public static QuotaProtos.TimedQuota toTimedQuota(final long limit, final TimeUnit timeUnit,
3106       final QuotaScope scope) {
3107     return QuotaProtos.TimedQuota.newBuilder().setSoftLimit(limit)
3108         .setTimeUnit(toProtoTimeUnit(timeUnit)).setScope(toProtoQuotaScope(scope)).build();
3109   }
3110 
3111   /**
3112    * Generates a marker for the WAL so that we propagate the notion of a bulk region load
3113    * throughout the WAL.
3114    *
3115    * @param tableName         The tableName into which the bulk load is being imported into.
3116    * @param encodedRegionName Encoded region name of the region which is being bulk loaded.
3117    * @param storeFiles        A set of store files of a column family are bulk loaded.
3118    * @param bulkloadSeqId     sequence ID (by a force flush) used to create bulk load hfile
3119    *                          name
3120    * @return The WAL log marker for bulk loads.
3121    */
3122   public static WALProtos.BulkLoadDescriptor toBulkLoadDescriptor(TableName tableName,
3123       ByteString encodedRegionName, Map<byte[], List<Path>> storeFiles, long bulkloadSeqId) {
3124     BulkLoadDescriptor.Builder desc = BulkLoadDescriptor.newBuilder()
3125         .setTableName(ProtobufUtil.toProtoTableName(tableName))
3126         .setEncodedRegionName(encodedRegionName).setBulkloadSeqNum(bulkloadSeqId);
3127 
3128     for (Map.Entry<byte[], List<Path>> entry : storeFiles.entrySet()) {
3129       WALProtos.StoreDescriptor.Builder builder = StoreDescriptor.newBuilder()
3130           .setFamilyName(ByteStringer.wrap(entry.getKey()))
3131           .setStoreHomeDir(Bytes.toString(entry.getKey())); // relative to region
3132       for (Path path : entry.getValue()) {
3133         builder.addStoreFile(path.getName());
3134       }
3135       desc.addStores(builder);
3136     }
3137 
3138     return desc.build();
3139   }
3140 
3141   /**
3142    * This version of protobuf's mergeDelimitedFrom avoids the hard-coded 64MB limit for decoding
3143    * buffers
3144    * @param builder current message builder
3145    * @param in Inputsream with delimited protobuf data
3146    * @throws IOException
3147    */
3148   public static void mergeDelimitedFrom(Message.Builder builder, InputStream in)
3149     throws IOException {
3150     // This used to be builder.mergeDelimitedFrom(in);
3151     // but is replaced to allow us to bump the protobuf size limit.
3152     final int firstByte = in.read();
3153     if (firstByte != -1) {
3154       final int size = CodedInputStream.readRawVarint32(firstByte, in);
3155       final InputStream limitedInput = new LimitInputStream(in, size);
3156       final CodedInputStream codedInput = CodedInputStream.newInstance(limitedInput);
3157       codedInput.setSizeLimit(size);
3158       builder.mergeFrom(codedInput);
3159       codedInput.checkLastTagWas(0);
3160     }
3161   }
3162 
3163   /**
3164    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3165    * buffers where the message size is known
3166    * @param builder current message builder
3167    * @param in InputStream containing protobuf data
3168    * @param size known size of protobuf data
3169    * @throws IOException 
3170    */
3171   public static void mergeFrom(Message.Builder builder, InputStream in, int size)
3172       throws IOException {
3173     final CodedInputStream codedInput = CodedInputStream.newInstance(in);
3174     codedInput.setSizeLimit(size);
3175     builder.mergeFrom(codedInput);
3176     codedInput.checkLastTagWas(0);
3177   }
3178 
3179   /**
3180    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3181    * buffers where the message size is not known
3182    * @param builder current message builder
3183    * @param in InputStream containing protobuf data
3184    * @throws IOException 
3185    */
3186   public static void mergeFrom(Message.Builder builder, InputStream in)
3187       throws IOException {
3188     final CodedInputStream codedInput = CodedInputStream.newInstance(in);
3189     codedInput.setSizeLimit(Integer.MAX_VALUE);
3190     builder.mergeFrom(codedInput);
3191     codedInput.checkLastTagWas(0);
3192   }
3193 
3194   /**
3195    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3196    * buffers when working with ByteStrings
3197    * @param builder current message builder
3198    * @param bs ByteString containing the 
3199    * @throws IOException 
3200    */
3201   public static void mergeFrom(Message.Builder builder, ByteString bs) throws IOException {
3202     final CodedInputStream codedInput = bs.newCodedInput();
3203     codedInput.setSizeLimit(bs.size());
3204     builder.mergeFrom(codedInput);
3205     codedInput.checkLastTagWas(0);
3206   }
3207 
3208   /**
3209    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3210    * buffers when working with byte arrays
3211    * @param builder current message builder
3212    * @param b byte array
3213    * @throws IOException 
3214    */
3215   public static void mergeFrom(Message.Builder builder, byte[] b) throws IOException {
3216     final CodedInputStream codedInput = CodedInputStream.newInstance(b);
3217     codedInput.setSizeLimit(b.length);
3218     builder.mergeFrom(codedInput);
3219     codedInput.checkLastTagWas(0);
3220   }
3221 
3222   /**
3223    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3224    * buffers when working with byte arrays
3225    * @param builder current message builder
3226    * @param b byte array
3227    * @param offset
3228    * @param length
3229    * @throws IOException
3230    */
3231   public static void mergeFrom(Message.Builder builder, byte[] b, int offset, int length)
3232       throws IOException {
3233     final CodedInputStream codedInput = CodedInputStream.newInstance(b, offset, length);
3234     codedInput.setSizeLimit(length);
3235     builder.mergeFrom(codedInput);
3236     codedInput.checkLastTagWas(0);
3237   }
3238 
3239   public static ReplicationLoadSink toReplicationLoadSink(
3240       ClusterStatusProtos.ReplicationLoadSink cls) {
3241     return new ReplicationLoadSink(cls.getAgeOfLastAppliedOp(), cls.getTimeStampsOfLastAppliedOp());
3242   }
3243 
3244   public static ReplicationLoadSource toReplicationLoadSource(
3245       ClusterStatusProtos.ReplicationLoadSource cls) {
3246     return new ReplicationLoadSource(cls.getPeerID(), cls.getAgeOfLastShippedOp(),
3247         cls.getSizeOfLogQueue(), cls.getTimeStampOfLastShippedOp(), cls.getReplicationLag());
3248   }
3249 
3250   public static List<ReplicationLoadSource> toReplicationLoadSourceList(
3251       List<ClusterStatusProtos.ReplicationLoadSource> clsList) {
3252     ArrayList<ReplicationLoadSource> rlsList = new ArrayList<ReplicationLoadSource>();
3253     for (ClusterStatusProtos.ReplicationLoadSource cls : clsList) {
3254       rlsList.add(toReplicationLoadSource(cls));
3255     }
3256     return rlsList;
3257   }
3258 
3259   /**
3260    * Get a protocol buffer VersionInfo
3261    *
3262    * @return the converted protocol buffer VersionInfo
3263    */
3264   public static HBaseProtos.VersionInfo getVersionInfo() {
3265     HBaseProtos.VersionInfo.Builder builder = HBaseProtos.VersionInfo.newBuilder();
3266     builder.setVersion(VersionInfo.getVersion());
3267     builder.setUrl(VersionInfo.getUrl());
3268     builder.setRevision(VersionInfo.getRevision());
3269     builder.setUser(VersionInfo.getUser());
3270     builder.setDate(VersionInfo.getDate());
3271     builder.setSrcChecksum(VersionInfo.getSrcChecksum());
3272     return builder.build();
3273   }
3274 
3275   /**
3276    * Convert SecurityCapabilitiesResponse.Capability to SecurityCapability
3277    * @param capabilities capabilities returned in the SecurityCapabilitiesResponse message
3278    * @return the converted list of SecurityCapability elements
3279    */
3280   public static List<SecurityCapability> toSecurityCapabilityList(
3281       List<MasterProtos.SecurityCapabilitiesResponse.Capability> capabilities) {
3282     List<SecurityCapability> scList = new ArrayList<>(capabilities.size());
3283     for (MasterProtos.SecurityCapabilitiesResponse.Capability c: capabilities) {
3284       try {
3285         scList.add(SecurityCapability.valueOf(c.getNumber()));
3286       } catch (IllegalArgumentException e) {
3287         // Unknown capability, just ignore it. We don't understand the new capability
3288         // but don't care since by definition we cannot take advantage of it.
3289       }
3290     }
3291     return scList;
3292   }
3293 
3294   private static HBaseProtos.TimeRange.Builder timeRangeToProto(TimeRange timeRange) {
3295     HBaseProtos.TimeRange.Builder timeRangeBuilder =
3296         HBaseProtos.TimeRange.newBuilder();
3297     timeRangeBuilder.setFrom(timeRange.getMin());
3298     timeRangeBuilder.setTo(timeRange.getMax());
3299     return timeRangeBuilder;
3300   }
3301 
3302   private static TimeRange protoToTimeRange(HBaseProtos.TimeRange timeRange) throws IOException {
3303       long minStamp = 0;
3304       long maxStamp = Long.MAX_VALUE;
3305       if (timeRange.hasFrom()) {
3306         minStamp = timeRange.getFrom();
3307       }
3308       if (timeRange.hasTo()) {
3309         maxStamp = timeRange.getTo();
3310       }
3311     return new TimeRange(minStamp, maxStamp);
3312   }
3313 
3314 }