View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.protobuf;
19  
20  
21  import static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME;
22  
23  import java.io.ByteArrayOutputStream;
24  import java.io.IOException;
25  import java.io.InputStream;
26  import java.lang.reflect.Constructor;
27  import java.lang.reflect.InvocationTargetException;
28  import java.lang.reflect.Method;
29  import java.lang.reflect.ParameterizedType;
30  import java.lang.reflect.Type;
31  import java.nio.ByteBuffer;
32  import java.util.ArrayList;
33  import java.util.Collection;
34  import java.util.HashMap;
35  import java.util.List;
36  import java.util.Map;
37  import java.util.Map.Entry;
38  import java.util.NavigableSet;
39  import java.util.concurrent.TimeUnit;
40  
41  import org.apache.hadoop.conf.Configuration;
42  import org.apache.hadoop.fs.Path;
43  import org.apache.hadoop.hbase.Cell;
44  import org.apache.hadoop.hbase.CellScanner;
45  import org.apache.hadoop.hbase.CellUtil;
46  import org.apache.hadoop.hbase.DoNotRetryIOException;
47  import org.apache.hadoop.hbase.HBaseConfiguration;
48  import org.apache.hadoop.hbase.HConstants;
49  import org.apache.hadoop.hbase.HRegionInfo;
50  import org.apache.hadoop.hbase.HTableDescriptor;
51  import org.apache.hadoop.hbase.KeyValue;
52  import org.apache.hadoop.hbase.NamespaceDescriptor;
53  import org.apache.hadoop.hbase.ServerName;
54  import org.apache.hadoop.hbase.TableName;
55  import org.apache.hadoop.hbase.Tag;
56  import org.apache.hadoop.hbase.classification.InterfaceAudience;
57  import org.apache.hadoop.hbase.client.Append;
58  import org.apache.hadoop.hbase.client.Consistency;
59  import org.apache.hadoop.hbase.client.Delete;
60  import org.apache.hadoop.hbase.client.Durability;
61  import org.apache.hadoop.hbase.client.Get;
62  import org.apache.hadoop.hbase.client.Increment;
63  import org.apache.hadoop.hbase.client.Mutation;
64  import org.apache.hadoop.hbase.client.Put;
65  import org.apache.hadoop.hbase.client.Result;
66  import org.apache.hadoop.hbase.client.Scan;
67  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
68  import org.apache.hadoop.hbase.client.security.SecurityCapability;
69  import org.apache.hadoop.hbase.exceptions.DeserializationException;
70  import org.apache.hadoop.hbase.filter.ByteArrayComparable;
71  import org.apache.hadoop.hbase.filter.Filter;
72  import org.apache.hadoop.hbase.io.LimitInputStream;
73  import org.apache.hadoop.hbase.io.TimeRange;
74  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
75  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService;
76  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
77  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
78  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
79  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
80  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
81  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
82  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
83  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
84  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
85  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
86  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
87  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
88  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
89  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
90  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
91  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest;
92  import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
93  import org.apache.hadoop.hbase.protobuf.generated.CellProtos;
94  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
95  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
96  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
97  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
98  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Column;
99  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall;
100 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
101 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
102 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
103 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
104 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
105 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue;
106 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue.QualifierValue;
107 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.DeleteType;
108 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
109 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
110 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
111 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
112 import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
113 import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
114 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
115 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
116 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo;
117 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
118 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
119 import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
120 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos;
121 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
122 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
123 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService;
124 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos;
125 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
126 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
127 import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
128 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
129 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
130 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
131 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
132 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor.EventType;
133 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.BulkLoadDescriptor;
134 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
135 import org.apache.hadoop.hbase.quotas.QuotaScope;
136 import org.apache.hadoop.hbase.quotas.QuotaType;
137 import org.apache.hadoop.hbase.quotas.ThrottleType;
138 import org.apache.hadoop.hbase.replication.ReplicationLoadSink;
139 import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
140 import org.apache.hadoop.hbase.security.access.Permission;
141 import org.apache.hadoop.hbase.security.access.TablePermission;
142 import org.apache.hadoop.hbase.security.access.UserPermission;
143 import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier;
144 import org.apache.hadoop.hbase.security.visibility.Authorizations;
145 import org.apache.hadoop.hbase.security.visibility.CellVisibility;
146 import org.apache.hadoop.hbase.util.ByteStringer;
147 import org.apache.hadoop.hbase.util.Bytes;
148 import org.apache.hadoop.hbase.util.DynamicClassLoader;
149 import org.apache.hadoop.hbase.util.ExceptionUtil;
150 import org.apache.hadoop.hbase.util.Methods;
151 import org.apache.hadoop.hbase.util.Pair;
152 import org.apache.hadoop.hbase.util.VersionInfo;
153 import org.apache.hadoop.io.Text;
154 import org.apache.hadoop.ipc.RemoteException;
155 import org.apache.hadoop.security.token.Token;
156 
157 import com.google.common.collect.ArrayListMultimap;
158 import com.google.common.collect.ListMultimap;
159 import com.google.common.collect.Lists;
160 import com.google.protobuf.ByteString;
161 import com.google.protobuf.CodedInputStream;
162 import com.google.protobuf.InvalidProtocolBufferException;
163 import com.google.protobuf.Message;
164 import com.google.protobuf.Parser;
165 import com.google.protobuf.RpcChannel;
166 import com.google.protobuf.RpcController;
167 import com.google.protobuf.Service;
168 import com.google.protobuf.ServiceException;
169 import com.google.protobuf.TextFormat;
170 
171 /**
172  * Protobufs utility.
173  */
174 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="DP_CREATE_CLASSLOADER_INSIDE_DO_PRIVILEGED",
175   justification="None. Address sometime.")
176 @InterfaceAudience.Private // TODO: some clients (Hive, etc) use this class
177 public final class ProtobufUtil {
178 
179   private ProtobufUtil() {
180   }
181 
182   /**
183    * Primitive type to class mapping.
184    */
185   private final static Map<String, Class<?>>
186     PRIMITIVES = new HashMap<String, Class<?>>();
187 
188 
189   /**
190    * Many results are simple: no cell, exists true or false. To save on object creations,
191    *  we reuse them across calls.
192    */
193   private final static Cell[] EMPTY_CELL_ARRAY = new Cell[]{};
194   private final static Result EMPTY_RESULT = Result.create(EMPTY_CELL_ARRAY);
195   private final static Result EMPTY_RESULT_EXISTS_TRUE = Result.create(null, true);
196   private final static Result EMPTY_RESULT_EXISTS_FALSE = Result.create(null, false);
197   private final static Result EMPTY_RESULT_STALE = Result.create(EMPTY_CELL_ARRAY, null, true);
198   private final static Result EMPTY_RESULT_EXISTS_TRUE_STALE
199     = Result.create((Cell[])null, true, true);
200   private final static Result EMPTY_RESULT_EXISTS_FALSE_STALE
201     = Result.create((Cell[])null, false, true);
202 
203   private final static ClientProtos.Result EMPTY_RESULT_PB;
204   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_TRUE;
205   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_FALSE;
206   private final static ClientProtos.Result EMPTY_RESULT_PB_STALE;
207   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_TRUE_STALE;
208   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_FALSE_STALE;
209 
210 
211   static {
212     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
213 
214     builder.setExists(true);
215     builder.setAssociatedCellCount(0);
216     EMPTY_RESULT_PB_EXISTS_TRUE =  builder.build();
217 
218     builder.setStale(true);
219     EMPTY_RESULT_PB_EXISTS_TRUE_STALE = builder.build();
220     builder.clear();
221 
222     builder.setExists(false);
223     builder.setAssociatedCellCount(0);
224     EMPTY_RESULT_PB_EXISTS_FALSE =  builder.build();
225     builder.setStale(true);
226     EMPTY_RESULT_PB_EXISTS_FALSE_STALE = builder.build();
227 
228     builder.clear();
229     builder.setAssociatedCellCount(0);
230     EMPTY_RESULT_PB =  builder.build();
231     builder.setStale(true);
232     EMPTY_RESULT_PB_STALE = builder.build();
233   }
234 
235   /**
236    * Dynamic class loader to load filter/comparators
237    */
238   private final static ClassLoader CLASS_LOADER;
239 
240   static {
241     ClassLoader parent = ProtobufUtil.class.getClassLoader();
242     Configuration conf = HBaseConfiguration.create();
243     CLASS_LOADER = new DynamicClassLoader(conf, parent);
244 
245     PRIMITIVES.put(Boolean.TYPE.getName(), Boolean.TYPE);
246     PRIMITIVES.put(Byte.TYPE.getName(), Byte.TYPE);
247     PRIMITIVES.put(Character.TYPE.getName(), Character.TYPE);
248     PRIMITIVES.put(Short.TYPE.getName(), Short.TYPE);
249     PRIMITIVES.put(Integer.TYPE.getName(), Integer.TYPE);
250     PRIMITIVES.put(Long.TYPE.getName(), Long.TYPE);
251     PRIMITIVES.put(Float.TYPE.getName(), Float.TYPE);
252     PRIMITIVES.put(Double.TYPE.getName(), Double.TYPE);
253     PRIMITIVES.put(Void.TYPE.getName(), Void.TYPE);
254   }
255 
256   /**
257    * Magic we put ahead of a serialized protobuf message.
258    * For example, all znode content is protobuf messages with the below magic
259    * for preamble.
260    */
261   public static final byte [] PB_MAGIC = new byte [] {'P', 'B', 'U', 'F'};
262   private static final String PB_MAGIC_STR = Bytes.toString(PB_MAGIC);
263 
264   /**
265    * Prepend the passed bytes with four bytes of magic, {@link #PB_MAGIC}, to flag what
266    * follows as a protobuf in hbase.  Prepend these bytes to all content written to znodes, etc.
267    * @param bytes Bytes to decorate
268    * @return The passed <code>bytes</code> with magic prepended (Creates a new
269    * byte array that is <code>bytes.length</code> plus {@link #PB_MAGIC}.length.
270    */
271   public static byte [] prependPBMagic(final byte [] bytes) {
272     return Bytes.add(PB_MAGIC, bytes);
273   }
274 
275   /**
276    * @param bytes Bytes to check.
277    * @return True if passed <code>bytes</code> has {@link #PB_MAGIC} for a prefix.
278    */
279   public static boolean isPBMagicPrefix(final byte [] bytes) {
280     if (bytes == null) return false;
281     return isPBMagicPrefix(bytes, 0, bytes.length);
282   }
283 
284   /**
285    * @param bytes Bytes to check.
286    * @param offset offset to start at
287    * @param len length to use
288    * @return True if passed <code>bytes</code> has {@link #PB_MAGIC} for a prefix.
289    */
290   public static boolean isPBMagicPrefix(final byte [] bytes, int offset, int len) {
291     if (bytes == null || len < PB_MAGIC.length) return false;
292     return Bytes.compareTo(PB_MAGIC, 0, PB_MAGIC.length, bytes, offset, PB_MAGIC.length) == 0;
293   }
294 
295   /**
296    * @param bytes bytes to check
297    * @throws DeserializationException if we are missing the pb magic prefix
298    */
299   public static void expectPBMagicPrefix(final byte [] bytes) throws DeserializationException {
300     if (!isPBMagicPrefix(bytes)) {
301       throw new DeserializationException("Missing pb magic " + PB_MAGIC_STR + " prefix");
302     }
303   }
304 
305   /**
306    * @return Length of {@link #PB_MAGIC}
307    */
308   public static int lengthOfPBMagic() {
309     return PB_MAGIC.length;
310   }
311 
312   /**
313    * Return the IOException thrown by the remote server wrapped in
314    * ServiceException as cause.
315    *
316    * @param se ServiceException that wraps IO exception thrown by the server
317    * @return Exception wrapped in ServiceException or
318    *   a new IOException that wraps the unexpected ServiceException.
319    */
320   public static IOException getRemoteException(ServiceException se) {
321     Throwable e = se.getCause();
322     if (e == null) {
323       return new IOException(se);
324     }
325     if (ExceptionUtil.isInterrupt(e)) {
326       return ExceptionUtil.asInterrupt(e);
327     }
328     if (e instanceof RemoteException) {
329       e = ((RemoteException) e).unwrapRemoteException();
330     }
331     return e instanceof IOException ? (IOException) e : new IOException(se);
332   }
333 
334   /**
335    * Convert a ServerName to a protocol buffer ServerName
336    *
337    * @param serverName the ServerName to convert
338    * @return the converted protocol buffer ServerName
339    * @see #toServerName(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName)
340    */
341   public static HBaseProtos.ServerName
342       toServerName(final ServerName serverName) {
343     if (serverName == null) return null;
344     HBaseProtos.ServerName.Builder builder =
345       HBaseProtos.ServerName.newBuilder();
346     builder.setHostName(serverName.getHostname());
347     if (serverName.getPort() >= 0) {
348       builder.setPort(serverName.getPort());
349     }
350     if (serverName.getStartcode() >= 0) {
351       builder.setStartCode(serverName.getStartcode());
352     }
353     return builder.build();
354   }
355 
356   /**
357    * Convert a protocol buffer ServerName to a ServerName
358    *
359    * @param proto the protocol buffer ServerName to convert
360    * @return the converted ServerName
361    */
362   public static ServerName toServerName(final HBaseProtos.ServerName proto) {
363     if (proto == null) return null;
364     String hostName = proto.getHostName();
365     long startCode = -1;
366     int port = -1;
367     if (proto.hasPort()) {
368       port = proto.getPort();
369     }
370     if (proto.hasStartCode()) {
371       startCode = proto.getStartCode();
372     }
373     return ServerName.valueOf(hostName, port, startCode);
374   }
375 
376   /**
377    * Get HTableDescriptor[] from GetTableDescriptorsResponse protobuf
378    *
379    * @param proto the GetTableDescriptorsResponse
380    * @return HTableDescriptor[]
381    */
382   public static HTableDescriptor[] getHTableDescriptorArray(GetTableDescriptorsResponse proto) {
383     if (proto == null) return null;
384 
385     HTableDescriptor[] ret = new HTableDescriptor[proto.getTableSchemaCount()];
386     for (int i = 0; i < proto.getTableSchemaCount(); ++i) {
387       ret[i] = HTableDescriptor.convert(proto.getTableSchema(i));
388     }
389     return ret;
390   }
391 
392   /**
393    * get the split keys in form "byte [][]" from a CreateTableRequest proto
394    *
395    * @param proto the CreateTableRequest
396    * @return the split keys
397    */
398   public static byte [][] getSplitKeysArray(final CreateTableRequest proto) {
399     byte [][] splitKeys = new byte[proto.getSplitKeysCount()][];
400     for (int i = 0; i < proto.getSplitKeysCount(); ++i) {
401       splitKeys[i] = proto.getSplitKeys(i).toByteArray();
402     }
403     return splitKeys;
404   }
405 
406   /**
407    * Convert a protobuf Durability into a client Durability
408    */
409   public static Durability toDurability(
410       final ClientProtos.MutationProto.Durability proto) {
411     switch(proto) {
412     case USE_DEFAULT:
413       return Durability.USE_DEFAULT;
414     case SKIP_WAL:
415       return Durability.SKIP_WAL;
416     case ASYNC_WAL:
417       return Durability.ASYNC_WAL;
418     case SYNC_WAL:
419       return Durability.SYNC_WAL;
420     case FSYNC_WAL:
421       return Durability.FSYNC_WAL;
422     default:
423       return Durability.USE_DEFAULT;
424     }
425   }
426 
427   /**
428    * Convert a client Durability into a protbuf Durability
429    */
430   public static ClientProtos.MutationProto.Durability toDurability(
431       final Durability d) {
432     switch(d) {
433     case USE_DEFAULT:
434       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
435     case SKIP_WAL:
436       return ClientProtos.MutationProto.Durability.SKIP_WAL;
437     case ASYNC_WAL:
438       return ClientProtos.MutationProto.Durability.ASYNC_WAL;
439     case SYNC_WAL:
440       return ClientProtos.MutationProto.Durability.SYNC_WAL;
441     case FSYNC_WAL:
442       return ClientProtos.MutationProto.Durability.FSYNC_WAL;
443     default:
444       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
445     }
446   }
447 
448   /**
449    * Convert a protocol buffer Get to a client Get
450    *
451    * @param proto the protocol buffer Get to convert
452    * @return the converted client Get
453    * @throws IOException
454    */
455   public static Get toGet(
456       final ClientProtos.Get proto) throws IOException {
457     if (proto == null) return null;
458     byte[] row = proto.getRow().toByteArray();
459     Get get = new Get(row);
460     if (proto.hasCacheBlocks()) {
461       get.setCacheBlocks(proto.getCacheBlocks());
462     }
463     if (proto.hasMaxVersions()) {
464       get.setMaxVersions(proto.getMaxVersions());
465     }
466     if (proto.hasStoreLimit()) {
467       get.setMaxResultsPerColumnFamily(proto.getStoreLimit());
468     }
469     if (proto.hasStoreOffset()) {
470       get.setRowOffsetPerColumnFamily(proto.getStoreOffset());
471     }
472     if (proto.getCfTimeRangeCount() > 0) {
473       for (HBaseProtos.ColumnFamilyTimeRange cftr : proto.getCfTimeRangeList()) {
474         TimeRange timeRange = protoToTimeRange(cftr.getTimeRange());
475         get.setColumnFamilyTimeRange(cftr.getColumnFamily().toByteArray(),
476             timeRange.getMin(), timeRange.getMax());
477       }
478     }
479     if (proto.hasTimeRange()) {
480       TimeRange timeRange = protoToTimeRange(proto.getTimeRange());
481       get.setTimeRange(timeRange.getMin(), timeRange.getMax());
482     }
483     if (proto.hasFilter()) {
484       FilterProtos.Filter filter = proto.getFilter();
485       get.setFilter(ProtobufUtil.toFilter(filter));
486     }
487     for (NameBytesPair attribute: proto.getAttributeList()) {
488       get.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
489     }
490     if (proto.getColumnCount() > 0) {
491       for (Column column: proto.getColumnList()) {
492         byte[] family = column.getFamily().toByteArray();
493         if (column.getQualifierCount() > 0) {
494           for (ByteString qualifier: column.getQualifierList()) {
495             get.addColumn(family, qualifier.toByteArray());
496           }
497         } else {
498           get.addFamily(family);
499         }
500       }
501     }
502     if (proto.hasExistenceOnly() && proto.getExistenceOnly()){
503       get.setCheckExistenceOnly(true);
504     }
505     if (proto.hasClosestRowBefore() && proto.getClosestRowBefore()){
506       get.setClosestRowBefore(true);
507     }
508     if (proto.hasConsistency()) {
509       get.setConsistency(toConsistency(proto.getConsistency()));
510     }
511     return get;
512   }
513 
514   public static Consistency toConsistency(ClientProtos.Consistency consistency) {
515     switch (consistency) {
516       case STRONG : return Consistency.STRONG;
517       case TIMELINE : return Consistency.TIMELINE;
518       default : return Consistency.STRONG;
519     }
520   }
521 
522   public static ClientProtos.Consistency toConsistency(Consistency consistency) {
523     switch (consistency) {
524       case STRONG : return ClientProtos.Consistency.STRONG;
525       case TIMELINE : return ClientProtos.Consistency.TIMELINE;
526       default : return ClientProtos.Consistency.STRONG;
527     }
528   }
529 
530   /**
531    * Convert a protocol buffer Mutate to a Put.
532    *
533    * @param proto The protocol buffer MutationProto to convert
534    * @return A client Put.
535    * @throws IOException
536    */
537   public static Put toPut(final MutationProto proto)
538   throws IOException {
539     return toPut(proto, null);
540   }
541 
542   /**
543    * Convert a protocol buffer Mutate to a Put.
544    *
545    * @param proto The protocol buffer MutationProto to convert
546    * @param cellScanner If non-null, the Cell data that goes with this proto.
547    * @return A client Put.
548    * @throws IOException
549    */
550   public static Put toPut(final MutationProto proto, final CellScanner cellScanner)
551   throws IOException {
552     // TODO: Server-side at least why do we convert back to the Client types?  Why not just pb it?
553     MutationType type = proto.getMutateType();
554     assert type == MutationType.PUT: type.name();
555     long timestamp = proto.hasTimestamp()? proto.getTimestamp(): HConstants.LATEST_TIMESTAMP;
556     Put put = proto.hasRow() ? new Put(proto.getRow().toByteArray(), timestamp) : null;
557     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
558     if (cellCount > 0) {
559       // The proto has metadata only and the data is separate to be found in the cellScanner.
560       if (cellScanner == null) {
561         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
562             toShortString(proto));
563       }
564       for (int i = 0; i < cellCount; i++) {
565         if (!cellScanner.advance()) {
566           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
567             " no cell returned: " + toShortString(proto));
568         }
569         Cell cell = cellScanner.current();
570         if (put == null) {
571           put = new Put(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
572         }
573         put.add(cell);
574       }
575     } else {
576       if (put == null) {
577         throw new IllegalArgumentException("row cannot be null");
578       }
579       // The proto has the metadata and the data itself
580       for (ColumnValue column: proto.getColumnValueList()) {
581         byte[] family = column.getFamily().toByteArray();
582         for (QualifierValue qv: column.getQualifierValueList()) {
583           if (!qv.hasValue()) {
584             throw new DoNotRetryIOException(
585                 "Missing required field: qualifier value");
586           }
587           ByteBuffer qualifier =
588               qv.hasQualifier() ? qv.getQualifier().asReadOnlyByteBuffer() : null;
589           ByteBuffer value =
590               qv.hasValue() ? qv.getValue().asReadOnlyByteBuffer() : null;
591           long ts = timestamp;
592           if (qv.hasTimestamp()) {
593             ts = qv.getTimestamp();
594           }
595           byte[] tags;
596           if (qv.hasTags()) {
597             tags = qv.getTags().toByteArray();
598             Object[] array = Tag.asList(tags, 0, (short)tags.length).toArray();
599             Tag[] tagArray = new Tag[array.length];
600             for(int i = 0; i< array.length; i++) {
601               tagArray[i] = (Tag)array[i];
602             }
603             if(qv.hasDeleteType()) {
604               byte[] qual = qv.hasQualifier() ? qv.getQualifier().toByteArray() : null;
605               put.add(new KeyValue(proto.getRow().toByteArray(), family, qual, ts,
606                   fromDeleteType(qv.getDeleteType()), null, tags));
607             } else {
608               put.addImmutable(family, qualifier, ts, value, tagArray);
609             }
610           } else {
611             if(qv.hasDeleteType()) {
612               byte[] qual = qv.hasQualifier() ? qv.getQualifier().toByteArray() : null;
613               put.add(new KeyValue(proto.getRow().toByteArray(), family, qual, ts,
614                   fromDeleteType(qv.getDeleteType())));
615             } else{
616               put.addImmutable(family, qualifier, ts, value);
617             }
618           }
619         }
620       }
621     }
622     put.setDurability(toDurability(proto.getDurability()));
623     for (NameBytesPair attribute: proto.getAttributeList()) {
624       put.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
625     }
626     return put;
627   }
628 
629   /**
630    * Convert a protocol buffer Mutate to a Delete
631    *
632    * @param proto the protocol buffer Mutate to convert
633    * @return the converted client Delete
634    * @throws IOException
635    */
636   public static Delete toDelete(final MutationProto proto)
637   throws IOException {
638     return toDelete(proto, null);
639   }
640 
641   /**
642    * Convert a protocol buffer Mutate to a Delete
643    *
644    * @param proto the protocol buffer Mutate to convert
645    * @param cellScanner if non-null, the data that goes with this delete.
646    * @return the converted client Delete
647    * @throws IOException
648    */
649   public static Delete toDelete(final MutationProto proto, final CellScanner cellScanner)
650   throws IOException {
651     MutationType type = proto.getMutateType();
652     assert type == MutationType.DELETE : type.name();
653     long timestamp = proto.hasTimestamp() ? proto.getTimestamp() : HConstants.LATEST_TIMESTAMP;
654     Delete delete = proto.hasRow() ? new Delete(proto.getRow().toByteArray(), timestamp) : null;
655     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
656     if (cellCount > 0) {
657       // The proto has metadata only and the data is separate to be found in the cellScanner.
658       if (cellScanner == null) {
659         // TextFormat should be fine for a Delete since it carries no data, just coordinates.
660         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
661           TextFormat.shortDebugString(proto));
662       }
663       for (int i = 0; i < cellCount; i++) {
664         if (!cellScanner.advance()) {
665           // TextFormat should be fine for a Delete since it carries no data, just coordinates.
666           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
667             " no cell returned: " + TextFormat.shortDebugString(proto));
668         }
669         Cell cell = cellScanner.current();
670         if (delete == null) {
671           delete =
672             new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
673         }
674         delete.addDeleteMarker(cell);
675       }
676     } else {
677       if (delete == null) {
678         throw new IllegalArgumentException("row cannot be null");
679       }
680       for (ColumnValue column: proto.getColumnValueList()) {
681         byte[] family = column.getFamily().toByteArray();
682         for (QualifierValue qv: column.getQualifierValueList()) {
683           DeleteType deleteType = qv.getDeleteType();
684           byte[] qualifier = null;
685           if (qv.hasQualifier()) {
686             qualifier = qv.getQualifier().toByteArray();
687           }
688           long ts = HConstants.LATEST_TIMESTAMP;
689           if (qv.hasTimestamp()) {
690             ts = qv.getTimestamp();
691           }
692           if (deleteType == DeleteType.DELETE_ONE_VERSION) {
693             delete.deleteColumn(family, qualifier, ts);
694           } else if (deleteType == DeleteType.DELETE_MULTIPLE_VERSIONS) {
695             delete.deleteColumns(family, qualifier, ts);
696           } else if (deleteType == DeleteType.DELETE_FAMILY_VERSION) {
697             delete.deleteFamilyVersion(family, ts);
698           } else {
699             delete.deleteFamily(family, ts);
700           }
701         }
702       }
703     }
704     delete.setDurability(toDurability(proto.getDurability()));
705     for (NameBytesPair attribute: proto.getAttributeList()) {
706       delete.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
707     }
708     return delete;
709   }
710 
711   /**
712    * Convert a protocol buffer Mutate to an Append
713    * @param cellScanner
714    * @param proto the protocol buffer Mutate to convert
715    * @return the converted client Append
716    * @throws IOException
717    */
718   public static Append toAppend(final MutationProto proto, final CellScanner cellScanner)
719   throws IOException {
720     MutationType type = proto.getMutateType();
721     assert type == MutationType.APPEND : type.name();
722     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
723     Append append = null;
724     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
725     if (cellCount > 0) {
726       // The proto has metadata only and the data is separate to be found in the cellScanner.
727       if (cellScanner == null) {
728         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
729           toShortString(proto));
730       }
731       for (int i = 0; i < cellCount; i++) {
732         if (!cellScanner.advance()) {
733           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
734             " no cell returned: " + toShortString(proto));
735         }
736         Cell cell = cellScanner.current();
737         if (append == null) {
738           append = new Append(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
739         }
740         append.add(cell);
741       }
742     } else {
743       append = new Append(row);
744       for (ColumnValue column: proto.getColumnValueList()) {
745         byte[] family = column.getFamily().toByteArray();
746         for (QualifierValue qv: column.getQualifierValueList()) {
747           byte[] qualifier = qv.getQualifier().toByteArray();
748           if (!qv.hasValue()) {
749             throw new DoNotRetryIOException(
750               "Missing required field: qualifier value");
751           }
752           byte[] value = qv.getValue().toByteArray();
753           byte[] tags = null;
754           if (qv.hasTags()) {
755             tags = qv.getTags().toByteArray();
756           }
757           append.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
758               KeyValue.Type.Put, value, tags));
759         }
760       }
761     }
762     append.setDurability(toDurability(proto.getDurability()));
763     for (NameBytesPair attribute: proto.getAttributeList()) {
764       append.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
765     }
766     return append;
767   }
768 
769   /**
770    * Convert a MutateRequest to Mutation
771    *
772    * @param proto the protocol buffer Mutate to convert
773    * @return the converted Mutation
774    * @throws IOException
775    */
776   public static Mutation toMutation(final MutationProto proto) throws IOException {
777     MutationType type = proto.getMutateType();
778     if (type == MutationType.APPEND) {
779       return toAppend(proto, null);
780     }
781     if (type == MutationType.DELETE) {
782       return toDelete(proto, null);
783     }
784     if (type == MutationType.PUT) {
785       return toPut(proto, null);
786     }
787     throw new IOException("Unknown mutation type " + type);
788   }
789 
790   /**
791    * Convert a protocol buffer Mutate to an Increment
792    *
793    * @param proto the protocol buffer Mutate to convert
794    * @return the converted client Increment
795    * @throws IOException
796    */
797   public static Increment toIncrement(final MutationProto proto, final CellScanner cellScanner)
798   throws IOException {
799     MutationType type = proto.getMutateType();
800     assert type == MutationType.INCREMENT : type.name();
801     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
802     Increment increment = null;
803     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
804     if (cellCount > 0) {
805       // The proto has metadata only and the data is separate to be found in the cellScanner.
806       if (cellScanner == null) {
807         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
808           TextFormat.shortDebugString(proto));
809       }
810       for (int i = 0; i < cellCount; i++) {
811         if (!cellScanner.advance()) {
812           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
813             " no cell returned: " + TextFormat.shortDebugString(proto));
814         }
815         Cell cell = cellScanner.current();
816         if (increment == null) {
817           increment = new Increment(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
818         }
819         increment.add(cell);
820       }
821     } else {
822       increment = new Increment(row);
823       for (ColumnValue column: proto.getColumnValueList()) {
824         byte[] family = column.getFamily().toByteArray();
825         for (QualifierValue qv: column.getQualifierValueList()) {
826           byte[] qualifier = qv.getQualifier().toByteArray();
827           if (!qv.hasValue()) {
828             throw new DoNotRetryIOException("Missing required field: qualifier value");
829           }
830           byte[] value = qv.getValue().toByteArray();
831           byte[] tags = null;
832           if (qv.hasTags()) {
833             tags = qv.getTags().toByteArray();
834           }
835           increment.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
836               KeyValue.Type.Put, value, tags));
837         }
838       }
839     }
840     if (proto.hasTimeRange()) {
841       TimeRange timeRange = protoToTimeRange(proto.getTimeRange());
842       increment.setTimeRange(timeRange.getMin(), timeRange.getMax());
843     }
844     increment.setDurability(toDurability(proto.getDurability()));
845     for (NameBytesPair attribute : proto.getAttributeList()) {
846       increment.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
847     }
848     return increment;
849   }
850 
851   /**
852    * Convert a client Scan to a protocol buffer Scan
853    *
854    * @param scan the client Scan to convert
855    * @return the converted protocol buffer Scan
856    * @throws IOException
857    */
858   public static ClientProtos.Scan toScan(
859       final Scan scan) throws IOException {
860     ClientProtos.Scan.Builder scanBuilder =
861       ClientProtos.Scan.newBuilder();
862     scanBuilder.setCacheBlocks(scan.getCacheBlocks());
863     if (scan.getBatch() > 0) {
864       scanBuilder.setBatchSize(scan.getBatch());
865     }
866     if (scan.getMaxResultSize() > 0) {
867       scanBuilder.setMaxResultSize(scan.getMaxResultSize());
868     }
869     if (scan.isSmall()) {
870       scanBuilder.setSmall(scan.isSmall());
871     }
872     if (scan.getAllowPartialResults()) {
873       scanBuilder.setAllowPartialResults(scan.getAllowPartialResults());
874     }
875     Boolean loadColumnFamiliesOnDemand = scan.getLoadColumnFamiliesOnDemandValue();
876     if (loadColumnFamiliesOnDemand != null) {
877       scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand.booleanValue());
878     }
879     scanBuilder.setMaxVersions(scan.getMaxVersions());
880     for (Entry<byte[], TimeRange> cftr : scan.getColumnFamilyTimeRange().entrySet()) {
881       HBaseProtos.ColumnFamilyTimeRange.Builder b = HBaseProtos.ColumnFamilyTimeRange.newBuilder();
882       b.setColumnFamily(ByteString.copyFrom(cftr.getKey()));
883       b.setTimeRange(timeRangeToProto(cftr.getValue()));
884       scanBuilder.addCfTimeRange(b);
885     }
886     TimeRange timeRange = scan.getTimeRange();
887     if (!timeRange.isAllTime()) {
888       HBaseProtos.TimeRange.Builder timeRangeBuilder =
889         HBaseProtos.TimeRange.newBuilder();
890       timeRangeBuilder.setFrom(timeRange.getMin());
891       timeRangeBuilder.setTo(timeRange.getMax());
892       scanBuilder.setTimeRange(timeRangeBuilder.build());
893     }
894     Map<String, byte[]> attributes = scan.getAttributesMap();
895     if (!attributes.isEmpty()) {
896       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
897       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
898         attributeBuilder.setName(attribute.getKey());
899         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
900         scanBuilder.addAttribute(attributeBuilder.build());
901       }
902     }
903     byte[] startRow = scan.getStartRow();
904     if (startRow != null && startRow.length > 0) {
905       scanBuilder.setStartRow(ByteStringer.wrap(startRow));
906     }
907     byte[] stopRow = scan.getStopRow();
908     if (stopRow != null && stopRow.length > 0) {
909       scanBuilder.setStopRow(ByteStringer.wrap(stopRow));
910     }
911     if (scan.hasFilter()) {
912       scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
913     }
914     if (scan.hasFamilies()) {
915       Column.Builder columnBuilder = Column.newBuilder();
916       for (Map.Entry<byte[],NavigableSet<byte []>>
917           family: scan.getFamilyMap().entrySet()) {
918         columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
919         NavigableSet<byte []> qualifiers = family.getValue();
920         columnBuilder.clearQualifier();
921         if (qualifiers != null && qualifiers.size() > 0) {
922           for (byte [] qualifier: qualifiers) {
923             columnBuilder.addQualifier(ByteStringer.wrap(qualifier));
924           }
925         }
926         scanBuilder.addColumn(columnBuilder.build());
927       }
928     }
929     if (scan.getMaxResultsPerColumnFamily() >= 0) {
930       scanBuilder.setStoreLimit(scan.getMaxResultsPerColumnFamily());
931     }
932     if (scan.getRowOffsetPerColumnFamily() > 0) {
933       scanBuilder.setStoreOffset(scan.getRowOffsetPerColumnFamily());
934     }
935     if (scan.isReversed()) {
936       scanBuilder.setReversed(scan.isReversed());
937     }
938     if (scan.getConsistency() == Consistency.TIMELINE) {
939       scanBuilder.setConsistency(toConsistency(scan.getConsistency()));
940     }
941     if (scan.getCaching() > 0) {
942       scanBuilder.setCaching(scan.getCaching());
943     }
944     return scanBuilder.build();
945   }
946 
947   /**
948    * Convert a protocol buffer Scan to a client Scan
949    *
950    * @param proto the protocol buffer Scan to convert
951    * @return the converted client Scan
952    * @throws IOException
953    */
954   public static Scan toScan(
955       final ClientProtos.Scan proto) throws IOException {
956     byte [] startRow = HConstants.EMPTY_START_ROW;
957     byte [] stopRow  = HConstants.EMPTY_END_ROW;
958     if (proto.hasStartRow()) {
959       startRow = proto.getStartRow().toByteArray();
960     }
961     if (proto.hasStopRow()) {
962       stopRow = proto.getStopRow().toByteArray();
963     }
964     Scan scan = new Scan(startRow, stopRow);
965     if (proto.hasCacheBlocks()) {
966       scan.setCacheBlocks(proto.getCacheBlocks());
967     }
968     if (proto.hasMaxVersions()) {
969       scan.setMaxVersions(proto.getMaxVersions());
970     }
971     if (proto.hasStoreLimit()) {
972       scan.setMaxResultsPerColumnFamily(proto.getStoreLimit());
973     }
974     if (proto.hasStoreOffset()) {
975       scan.setRowOffsetPerColumnFamily(proto.getStoreOffset());
976     }
977     if (proto.hasLoadColumnFamiliesOnDemand()) {
978       scan.setLoadColumnFamiliesOnDemand(proto.getLoadColumnFamiliesOnDemand());
979     }
980     if (proto.getCfTimeRangeCount() > 0) {
981       for (HBaseProtos.ColumnFamilyTimeRange cftr : proto.getCfTimeRangeList()) {
982         TimeRange timeRange = protoToTimeRange(cftr.getTimeRange());
983         scan.setColumnFamilyTimeRange(cftr.getColumnFamily().toByteArray(),
984             timeRange.getMin(), timeRange.getMax());
985       }
986     }
987     if (proto.hasTimeRange()) {
988       TimeRange timeRange = protoToTimeRange(proto.getTimeRange());
989       scan.setTimeRange(timeRange.getMin(), timeRange.getMax());
990     }
991     if (proto.hasFilter()) {
992       FilterProtos.Filter filter = proto.getFilter();
993       scan.setFilter(ProtobufUtil.toFilter(filter));
994     }
995     if (proto.hasBatchSize()) {
996       scan.setBatch(proto.getBatchSize());
997     }
998     if (proto.hasMaxResultSize()) {
999       scan.setMaxResultSize(proto.getMaxResultSize());
1000     }
1001     if (proto.hasSmall()) {
1002       scan.setSmall(proto.getSmall());
1003     }
1004     if (proto.hasAllowPartialResults()) {
1005       scan.setAllowPartialResults(proto.getAllowPartialResults());
1006     }
1007     for (NameBytesPair attribute: proto.getAttributeList()) {
1008       scan.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
1009     }
1010     if (proto.getColumnCount() > 0) {
1011       for (Column column: proto.getColumnList()) {
1012         byte[] family = column.getFamily().toByteArray();
1013         if (column.getQualifierCount() > 0) {
1014           for (ByteString qualifier: column.getQualifierList()) {
1015             scan.addColumn(family, qualifier.toByteArray());
1016           }
1017         } else {
1018           scan.addFamily(family);
1019         }
1020       }
1021     }
1022     if (proto.hasReversed()) {
1023       scan.setReversed(proto.getReversed());
1024     }
1025     if (proto.hasConsistency()) {
1026       scan.setConsistency(toConsistency(proto.getConsistency()));
1027     }
1028     if (proto.hasCaching()) {
1029       scan.setCaching(proto.getCaching());
1030     }
1031     return scan;
1032   }
1033 
1034   /**
1035    * Create a protocol buffer Get based on a client Get.
1036    *
1037    * @param get the client Get
1038    * @return a protocol buffer Get
1039    * @throws IOException
1040    */
1041   public static ClientProtos.Get toGet(
1042       final Get get) throws IOException {
1043     ClientProtos.Get.Builder builder =
1044       ClientProtos.Get.newBuilder();
1045     builder.setRow(ByteStringer.wrap(get.getRow()));
1046     builder.setCacheBlocks(get.getCacheBlocks());
1047     builder.setMaxVersions(get.getMaxVersions());
1048     if (get.getFilter() != null) {
1049       builder.setFilter(ProtobufUtil.toFilter(get.getFilter()));
1050     }
1051     for (Entry<byte[], TimeRange> cftr : get.getColumnFamilyTimeRange().entrySet()) {
1052       HBaseProtos.ColumnFamilyTimeRange.Builder b = HBaseProtos.ColumnFamilyTimeRange.newBuilder();
1053       b.setColumnFamily(ByteString.copyFrom(cftr.getKey()));
1054       b.setTimeRange(timeRangeToProto(cftr.getValue()));
1055       builder.addCfTimeRange(b);
1056     }
1057     TimeRange timeRange = get.getTimeRange();
1058     if (!timeRange.isAllTime()) {
1059       HBaseProtos.TimeRange.Builder timeRangeBuilder =
1060         HBaseProtos.TimeRange.newBuilder();
1061       timeRangeBuilder.setFrom(timeRange.getMin());
1062       timeRangeBuilder.setTo(timeRange.getMax());
1063       builder.setTimeRange(timeRangeBuilder.build());
1064     }
1065     Map<String, byte[]> attributes = get.getAttributesMap();
1066     if (!attributes.isEmpty()) {
1067       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1068       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
1069         attributeBuilder.setName(attribute.getKey());
1070         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1071         builder.addAttribute(attributeBuilder.build());
1072       }
1073     }
1074     if (get.hasFamilies()) {
1075       Column.Builder columnBuilder = Column.newBuilder();
1076       Map<byte[], NavigableSet<byte[]>> families = get.getFamilyMap();
1077       for (Map.Entry<byte[], NavigableSet<byte[]>> family: families.entrySet()) {
1078         NavigableSet<byte[]> qualifiers = family.getValue();
1079         columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1080         columnBuilder.clearQualifier();
1081         if (qualifiers != null && qualifiers.size() > 0) {
1082           for (byte[] qualifier: qualifiers) {
1083             columnBuilder.addQualifier(ByteStringer.wrap(qualifier));
1084           }
1085         }
1086         builder.addColumn(columnBuilder.build());
1087       }
1088     }
1089     if (get.getMaxResultsPerColumnFamily() >= 0) {
1090       builder.setStoreLimit(get.getMaxResultsPerColumnFamily());
1091     }
1092     if (get.getRowOffsetPerColumnFamily() > 0) {
1093       builder.setStoreOffset(get.getRowOffsetPerColumnFamily());
1094     }
1095     if (get.isCheckExistenceOnly()){
1096       builder.setExistenceOnly(true);
1097     }
1098     if (get.isClosestRowBefore()){
1099       builder.setClosestRowBefore(true);
1100     }
1101     if (get.getConsistency() != null && get.getConsistency() != Consistency.STRONG) {
1102       builder.setConsistency(toConsistency(get.getConsistency()));
1103     }
1104 
1105     return builder.build();
1106   }
1107 
1108   static void setTimeRange(final MutationProto.Builder builder, final TimeRange timeRange) {
1109     if (!timeRange.isAllTime()) {
1110       HBaseProtos.TimeRange.Builder timeRangeBuilder =
1111         HBaseProtos.TimeRange.newBuilder();
1112       timeRangeBuilder.setFrom(timeRange.getMin());
1113       timeRangeBuilder.setTo(timeRange.getMax());
1114       builder.setTimeRange(timeRangeBuilder.build());
1115     }
1116   }
1117 
1118   /**
1119    * Convert a client Increment to a protobuf Mutate.
1120    *
1121    * @param increment
1122    * @return the converted mutate
1123    */
1124   public static MutationProto toMutation(
1125     final Increment increment, final MutationProto.Builder builder, long nonce) {
1126     builder.setRow(ByteStringer.wrap(increment.getRow()));
1127     builder.setMutateType(MutationType.INCREMENT);
1128     builder.setDurability(toDurability(increment.getDurability()));
1129     if (nonce != HConstants.NO_NONCE) {
1130       builder.setNonce(nonce);
1131     }
1132     TimeRange timeRange = increment.getTimeRange();
1133     setTimeRange(builder, timeRange);
1134     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1135     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1136     for (Map.Entry<byte[], List<Cell>> family: increment.getFamilyCellMap().entrySet()) {
1137       columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1138       columnBuilder.clearQualifierValue();
1139       List<Cell> values = family.getValue();
1140       if (values != null && values.size() > 0) {
1141         for (Cell cell: values) {
1142           valueBuilder.clear();
1143           valueBuilder.setQualifier(ByteStringer.wrap(
1144               cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
1145           valueBuilder.setValue(ByteStringer.wrap(
1146               cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
1147           if (cell.getTagsLength() > 0) {
1148             valueBuilder.setTags(ByteStringer.wrap(cell.getTagsArray(),
1149                 cell.getTagsOffset(), cell.getTagsLength()));
1150           }
1151           columnBuilder.addQualifierValue(valueBuilder.build());
1152         }
1153       }
1154       builder.addColumnValue(columnBuilder.build());
1155     }
1156     Map<String, byte[]> attributes = increment.getAttributesMap();
1157     if (!attributes.isEmpty()) {
1158       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1159       for (Map.Entry<String, byte[]> attribute : attributes.entrySet()) {
1160         attributeBuilder.setName(attribute.getKey());
1161         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1162         builder.addAttribute(attributeBuilder.build());
1163       }
1164     }
1165     return builder.build();
1166   }
1167 
1168   public static MutationProto toMutation(final MutationType type, final Mutation mutation)
1169     throws IOException {
1170     return toMutation(type, mutation, HConstants.NO_NONCE);
1171   }
1172 
1173   /**
1174    * Create a protocol buffer Mutate based on a client Mutation
1175    *
1176    * @param type
1177    * @param mutation
1178    * @return a protobuf'd Mutation
1179    * @throws IOException
1180    */
1181   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1182     final long nonce) throws IOException {
1183     return toMutation(type, mutation, MutationProto.newBuilder(), nonce);
1184   }
1185 
1186   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1187       MutationProto.Builder builder) throws IOException {
1188     return toMutation(type, mutation, builder, HConstants.NO_NONCE);
1189   }
1190 
1191   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1192       MutationProto.Builder builder, long nonce)
1193   throws IOException {
1194     builder = getMutationBuilderAndSetCommonFields(type, mutation, builder);
1195     if (nonce != HConstants.NO_NONCE) {
1196       builder.setNonce(nonce);
1197     }
1198     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1199     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1200     for (Map.Entry<byte[],List<Cell>> family: mutation.getFamilyCellMap().entrySet()) {
1201       columnBuilder.clear();
1202       columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1203       for (Cell cell: family.getValue()) {
1204         valueBuilder.clear();
1205         valueBuilder.setQualifier(ByteStringer.wrap(
1206             cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
1207         valueBuilder.setValue(ByteStringer.wrap(
1208             cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
1209         valueBuilder.setTimestamp(cell.getTimestamp());
1210         if (type == MutationType.DELETE || (type == MutationType.PUT && CellUtil.isDelete(cell))) {
1211           KeyValue.Type keyValueType = KeyValue.Type.codeToType(cell.getTypeByte());
1212           valueBuilder.setDeleteType(toDeleteType(keyValueType));
1213         }
1214         columnBuilder.addQualifierValue(valueBuilder.build());
1215       }
1216       builder.addColumnValue(columnBuilder.build());
1217     }
1218     return builder.build();
1219   }
1220 
1221   /**
1222    * Create a protocol buffer MutationProto based on a client Mutation. Does NOT include data.
1223    * Understanding is that the Cell will be transported other than via protobuf.
1224    * @param type
1225    * @param mutation
1226    * @param builder
1227    * @return a protobuf'd Mutation
1228    * @throws IOException
1229    */
1230   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1231       final MutationProto.Builder builder)  throws IOException {
1232     return toMutationNoData(type, mutation, builder, HConstants.NO_NONCE);
1233   }
1234 
1235   /**
1236    * Create a protocol buffer MutationProto based on a client Mutation.  Does NOT include data.
1237    * Understanding is that the Cell will be transported other than via protobuf.
1238    * @param type
1239    * @param mutation
1240    * @return a protobuf'd Mutation
1241    * @throws IOException
1242    */
1243   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation)
1244   throws IOException {
1245     MutationProto.Builder builder =  MutationProto.newBuilder();
1246     return toMutationNoData(type, mutation, builder);
1247   }
1248 
1249   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1250       final MutationProto.Builder builder, long nonce) throws IOException {
1251     getMutationBuilderAndSetCommonFields(type, mutation, builder);
1252     builder.setAssociatedCellCount(mutation.size());
1253     if (mutation instanceof Increment) {
1254       setTimeRange(builder, ((Increment)mutation).getTimeRange());
1255     }
1256     if (nonce != HConstants.NO_NONCE) {
1257       builder.setNonce(nonce);
1258     }
1259     return builder.build();
1260   }
1261 
1262   /**
1263    * Code shared by {@link #toMutation(MutationType, Mutation)} and
1264    * {@link #toMutationNoData(MutationType, Mutation)}
1265    * @param type
1266    * @param mutation
1267    * @return A partly-filled out protobuf'd Mutation.
1268    */
1269   private static MutationProto.Builder getMutationBuilderAndSetCommonFields(final MutationType type,
1270       final Mutation mutation, MutationProto.Builder builder) {
1271     builder.setRow(ByteStringer.wrap(mutation.getRow()));
1272     builder.setMutateType(type);
1273     builder.setDurability(toDurability(mutation.getDurability()));
1274     builder.setTimestamp(mutation.getTimeStamp());
1275     Map<String, byte[]> attributes = mutation.getAttributesMap();
1276     if (!attributes.isEmpty()) {
1277       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1278       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
1279         attributeBuilder.setName(attribute.getKey());
1280         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1281         builder.addAttribute(attributeBuilder.build());
1282       }
1283     }
1284     return builder;
1285   }
1286 
1287   /**
1288    * Convert a client Result to a protocol buffer Result
1289    *
1290    * @param result the client Result to convert
1291    * @return the converted protocol buffer Result
1292    */
1293   public static ClientProtos.Result toResult(final Result result) {
1294     if (result.getExists() != null) {
1295       return toResult(result.getExists(), result.isStale());
1296     }
1297 
1298     Cell[] cells = result.rawCells();
1299     if (cells == null || cells.length == 0) {
1300       return result.isStale() ? EMPTY_RESULT_PB_STALE : EMPTY_RESULT_PB;
1301     }
1302 
1303     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1304     for (Cell c : cells) {
1305       builder.addCell(toCell(c));
1306     }
1307 
1308     builder.setStale(result.isStale());
1309     builder.setPartial(result.isPartial());
1310 
1311     return builder.build();
1312   }
1313 
1314   /**
1315    * Convert a client Result to a protocol buffer Result
1316    *
1317    * @param existence the client existence to send
1318    * @return the converted protocol buffer Result
1319    */
1320   public static ClientProtos.Result toResult(final boolean existence, boolean stale) {
1321     if (stale){
1322       return existence ? EMPTY_RESULT_PB_EXISTS_TRUE_STALE : EMPTY_RESULT_PB_EXISTS_FALSE_STALE;
1323     } else {
1324       return existence ? EMPTY_RESULT_PB_EXISTS_TRUE : EMPTY_RESULT_PB_EXISTS_FALSE;
1325     }
1326   }
1327 
1328   /**
1329    * Convert a client Result to a protocol buffer Result.
1330    * The pb Result does not include the Cell data.  That is for transport otherwise.
1331    *
1332    * @param result the client Result to convert
1333    * @return the converted protocol buffer Result
1334    */
1335   public static ClientProtos.Result toResultNoData(final Result result) {
1336     if (result.getExists() != null) return toResult(result.getExists(), result.isStale());
1337     int size = result.size();
1338     if (size == 0) return result.isStale() ? EMPTY_RESULT_PB_STALE : EMPTY_RESULT_PB;
1339     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1340     builder.setAssociatedCellCount(size);
1341     builder.setStale(result.isStale());
1342     return builder.build();
1343   }
1344 
1345   /**
1346    * Convert a protocol buffer Result to a client Result
1347    *
1348    * @param proto the protocol buffer Result to convert
1349    * @return the converted client Result
1350    */
1351   public static Result toResult(final ClientProtos.Result proto) {
1352     if (proto.hasExists()) {
1353       if (proto.getStale()) {
1354         return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE_STALE :EMPTY_RESULT_EXISTS_FALSE_STALE;
1355       }
1356       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1357     }
1358 
1359     List<CellProtos.Cell> values = proto.getCellList();
1360     if (values.isEmpty()){
1361       return proto.getStale() ? EMPTY_RESULT_STALE : EMPTY_RESULT;
1362     }
1363 
1364     List<Cell> cells = new ArrayList<Cell>(values.size());
1365     for (CellProtos.Cell c : values) {
1366       cells.add(toCell(c));
1367     }
1368     return Result.create(cells, null, proto.getStale(), proto.getPartial());
1369   }
1370 
1371   /**
1372    * Convert a protocol buffer Result to a client Result
1373    *
1374    * @param proto the protocol buffer Result to convert
1375    * @param scanner Optional cell scanner.
1376    * @return the converted client Result
1377    * @throws IOException
1378    */
1379   public static Result toResult(final ClientProtos.Result proto, final CellScanner scanner)
1380   throws IOException {
1381     List<CellProtos.Cell> values = proto.getCellList();
1382 
1383     if (proto.hasExists()) {
1384       if ((values != null && !values.isEmpty()) ||
1385           (proto.hasAssociatedCellCount() && proto.getAssociatedCellCount() > 0)) {
1386         throw new IllegalArgumentException("bad proto: exists with cells is no allowed " + proto);
1387       }
1388       if (proto.getStale()) {
1389         return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE_STALE :EMPTY_RESULT_EXISTS_FALSE_STALE;
1390       }
1391       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1392     }
1393 
1394     // TODO: Unit test that has some Cells in scanner and some in the proto.
1395     List<Cell> cells = null;
1396     if (proto.hasAssociatedCellCount()) {
1397       int count = proto.getAssociatedCellCount();
1398       cells = new ArrayList<Cell>(count + values.size());
1399       for (int i = 0; i < count; i++) {
1400         if (!scanner.advance()) throw new IOException("Failed get " + i + " of " + count);
1401         cells.add(scanner.current());
1402       }
1403     }
1404 
1405     if (!values.isEmpty()){
1406       if (cells == null) cells = new ArrayList<Cell>(values.size());
1407       for (CellProtos.Cell c: values) {
1408         cells.add(toCell(c));
1409       }
1410     }
1411 
1412     return (cells == null || cells.isEmpty())
1413         ? (proto.getStale() ? EMPTY_RESULT_STALE : EMPTY_RESULT)
1414         : Result.create(cells, null, proto.getStale());
1415   }
1416 
1417 
1418   /**
1419    * Convert a ByteArrayComparable to a protocol buffer Comparator
1420    *
1421    * @param comparator the ByteArrayComparable to convert
1422    * @return the converted protocol buffer Comparator
1423    */
1424   public static ComparatorProtos.Comparator toComparator(ByteArrayComparable comparator) {
1425     ComparatorProtos.Comparator.Builder builder = ComparatorProtos.Comparator.newBuilder();
1426     builder.setName(comparator.getClass().getName());
1427     builder.setSerializedComparator(ByteStringer.wrap(comparator.toByteArray()));
1428     return builder.build();
1429   }
1430 
1431   /**
1432    * Convert a protocol buffer Comparator to a ByteArrayComparable
1433    *
1434    * @param proto the protocol buffer Comparator to convert
1435    * @return the converted ByteArrayComparable
1436    */
1437   @SuppressWarnings("unchecked")
1438   public static ByteArrayComparable toComparator(ComparatorProtos.Comparator proto)
1439   throws IOException {
1440     String type = proto.getName();
1441     String funcName = "parseFrom";
1442     byte [] value = proto.getSerializedComparator().toByteArray();
1443     try {
1444       Class<? extends ByteArrayComparable> c =
1445         (Class<? extends ByteArrayComparable>)Class.forName(type, true, CLASS_LOADER);
1446       Method parseFrom = c.getMethod(funcName, byte[].class);
1447       if (parseFrom == null) {
1448         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1449       }
1450       return (ByteArrayComparable)parseFrom.invoke(null, value);
1451     } catch (Exception e) {
1452       throw new IOException(e);
1453     }
1454   }
1455 
1456   /**
1457    * Convert a protocol buffer Filter to a client Filter
1458    *
1459    * @param proto the protocol buffer Filter to convert
1460    * @return the converted Filter
1461    */
1462   @SuppressWarnings("unchecked")
1463   public static Filter toFilter(FilterProtos.Filter proto) throws IOException {
1464     String type = proto.getName();
1465     final byte [] value = proto.getSerializedFilter().toByteArray();
1466     String funcName = "parseFrom";
1467     try {
1468       Class<? extends Filter> c =
1469         (Class<? extends Filter>)Class.forName(type, true, CLASS_LOADER);
1470       Method parseFrom = c.getMethod(funcName, byte[].class);
1471       if (parseFrom == null) {
1472         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1473       }
1474       return (Filter)parseFrom.invoke(c, value);
1475     } catch (Exception e) {
1476       // Either we couldn't instantiate the method object, or "parseFrom" failed.
1477       // In either case, let's not retry.
1478       throw new DoNotRetryIOException(e);
1479     }
1480   }
1481 
1482   /**
1483    * Convert a client Filter to a protocol buffer Filter
1484    *
1485    * @param filter the Filter to convert
1486    * @return the converted protocol buffer Filter
1487    */
1488   public static FilterProtos.Filter toFilter(Filter filter) throws IOException {
1489     FilterProtos.Filter.Builder builder = FilterProtos.Filter.newBuilder();
1490     builder.setName(filter.getClass().getName());
1491     builder.setSerializedFilter(ByteStringer.wrap(filter.toByteArray()));
1492     return builder.build();
1493   }
1494 
1495   /**
1496    * Convert a delete KeyValue type to protocol buffer DeleteType.
1497    *
1498    * @param type
1499    * @return protocol buffer DeleteType
1500    * @throws IOException
1501    */
1502   public static DeleteType toDeleteType(
1503       KeyValue.Type type) throws IOException {
1504     switch (type) {
1505     case Delete:
1506       return DeleteType.DELETE_ONE_VERSION;
1507     case DeleteColumn:
1508       return DeleteType.DELETE_MULTIPLE_VERSIONS;
1509     case DeleteFamily:
1510       return DeleteType.DELETE_FAMILY;
1511     case DeleteFamilyVersion:
1512       return DeleteType.DELETE_FAMILY_VERSION;
1513     default:
1514         throw new IOException("Unknown delete type: " + type);
1515     }
1516   }
1517 
1518   /**
1519    * Convert a protocol buffer DeleteType to delete KeyValue type.
1520    *
1521    * @param type The DeleteType
1522    * @return The type.
1523    * @throws IOException
1524    */
1525   public static KeyValue.Type fromDeleteType(
1526       DeleteType type) throws IOException {
1527     switch (type) {
1528     case DELETE_ONE_VERSION:
1529       return KeyValue.Type.Delete;
1530     case DELETE_MULTIPLE_VERSIONS:
1531       return KeyValue.Type.DeleteColumn;
1532     case DELETE_FAMILY:
1533       return KeyValue.Type.DeleteFamily;
1534     case DELETE_FAMILY_VERSION:
1535       return KeyValue.Type.DeleteFamilyVersion;
1536     default:
1537       throw new IOException("Unknown delete type: " + type);
1538     }
1539   }
1540 
1541   /**
1542    * Convert a stringified protocol buffer exception Parameter to a Java Exception
1543    *
1544    * @param parameter the protocol buffer Parameter to convert
1545    * @return the converted Exception
1546    * @throws IOException if failed to deserialize the parameter
1547    */
1548   @SuppressWarnings("unchecked")
1549   public static Throwable toException(final NameBytesPair parameter) throws IOException {
1550     if (parameter == null || !parameter.hasValue()) return null;
1551     String desc = parameter.getValue().toStringUtf8();
1552     String type = parameter.getName();
1553     try {
1554       Class<? extends Throwable> c =
1555         (Class<? extends Throwable>)Class.forName(type, true, CLASS_LOADER);
1556       Constructor<? extends Throwable> cn = null;
1557       try {
1558         cn = c.getDeclaredConstructor(String.class);
1559         return cn.newInstance(desc);
1560       } catch (NoSuchMethodException e) {
1561         // Could be a raw RemoteException. See HBASE-8987.
1562         cn = c.getDeclaredConstructor(String.class, String.class);
1563         return cn.newInstance(type, desc);
1564       }
1565     } catch (Exception e) {
1566       throw new IOException(e);
1567     }
1568   }
1569 
1570 // Start helpers for Client
1571 
1572   /**
1573    * A helper to get a row of the closet one before using client protocol.
1574    *
1575    * @param client
1576    * @param regionName
1577    * @param row
1578    * @param family
1579    * @return the row or the closestRowBefore if it doesn't exist
1580    * @throws IOException
1581    * @deprecated since 0.99 - use reversed scanner instead.
1582    */
1583   @Deprecated
1584   public static Result getRowOrBefore(final ClientService.BlockingInterface client,
1585       final byte[] regionName, final byte[] row,
1586       final byte[] family) throws IOException {
1587     GetRequest request =
1588       RequestConverter.buildGetRowOrBeforeRequest(
1589         regionName, row, family);
1590     try {
1591       GetResponse response = client.get(null, request);
1592       if (!response.hasResult()) return null;
1593       return toResult(response.getResult());
1594     } catch (ServiceException se) {
1595       throw getRemoteException(se);
1596     }
1597   }
1598 
1599   /**
1600    * A helper to bulk load a list of HFiles using client protocol.
1601    *
1602    * @param client
1603    * @param familyPaths
1604    * @param regionName
1605    * @param assignSeqNum
1606    * @return true if all are loaded
1607    * @throws IOException
1608    */
1609   public static boolean bulkLoadHFile(final ClientService.BlockingInterface client,
1610       final List<Pair<byte[], String>> familyPaths,
1611       final byte[] regionName, boolean assignSeqNum) throws IOException {
1612     BulkLoadHFileRequest request =
1613       RequestConverter.buildBulkLoadHFileRequest(familyPaths, regionName, assignSeqNum);
1614     try {
1615       BulkLoadHFileResponse response =
1616         client.bulkLoadHFile(null, request);
1617       return response.getLoaded();
1618     } catch (ServiceException se) {
1619       throw getRemoteException(se);
1620     }
1621   }
1622 
1623   public static CoprocessorServiceResponse execService(final RpcController controller,
1624       final ClientService.BlockingInterface client, final CoprocessorServiceCall call,
1625       final byte[] regionName) throws IOException {
1626     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1627         .setCall(call).setRegion(
1628             RequestConverter.buildRegionSpecifier(REGION_NAME, regionName)).build();
1629     try {
1630       CoprocessorServiceResponse response =
1631           client.execService(controller, request);
1632       return response;
1633     } catch (ServiceException se) {
1634       throw getRemoteException(se);
1635     }
1636   }
1637 
1638   public static CoprocessorServiceResponse execService(final RpcController controller,
1639     final MasterService.BlockingInterface client, final CoprocessorServiceCall call)
1640   throws IOException {
1641     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1642         .setCall(call).setRegion(
1643             RequestConverter.buildRegionSpecifier(REGION_NAME, HConstants.EMPTY_BYTE_ARRAY)).build();
1644     try {
1645       CoprocessorServiceResponse response =
1646           client.execMasterService(controller, request);
1647       return response;
1648     } catch (ServiceException se) {
1649       throw getRemoteException(se);
1650     }
1651   }
1652 
1653   /**
1654    * Make a region server endpoint call
1655    * @param client
1656    * @param call
1657    * @return CoprocessorServiceResponse
1658    * @throws IOException
1659    */
1660   public static CoprocessorServiceResponse execRegionServerService(
1661       final RpcController controller, final ClientService.BlockingInterface client,
1662       final CoprocessorServiceCall call)
1663       throws IOException {
1664     CoprocessorServiceRequest request =
1665         CoprocessorServiceRequest
1666             .newBuilder()
1667             .setCall(call)
1668             .setRegion(
1669               RequestConverter.buildRegionSpecifier(REGION_NAME, HConstants.EMPTY_BYTE_ARRAY))
1670             .build();
1671     try {
1672       CoprocessorServiceResponse response = client.execRegionServerService(controller, request);
1673       return response;
1674     } catch (ServiceException se) {
1675       throw getRemoteException(se);
1676     }
1677   }
1678 
1679   @SuppressWarnings("unchecked")
1680   public static <T extends Service> T newServiceStub(Class<T> service, RpcChannel channel)
1681       throws Exception {
1682     return (T)Methods.call(service, null, "newStub",
1683         new Class[]{ RpcChannel.class }, new Object[]{ channel });
1684   }
1685 
1686 // End helpers for Client
1687 // Start helpers for Admin
1688 
1689   /**
1690    * A helper to retrieve region info given a region name
1691    * using admin protocol.
1692    *
1693    * @param admin
1694    * @param regionName
1695    * @return the retrieved region info
1696    * @throws IOException
1697    */
1698   public static HRegionInfo getRegionInfo(final RpcController controller,
1699       final AdminService.BlockingInterface admin, final byte[] regionName) throws IOException {
1700     try {
1701       GetRegionInfoRequest request =
1702         RequestConverter.buildGetRegionInfoRequest(regionName);
1703       GetRegionInfoResponse response =
1704         admin.getRegionInfo(controller, request);
1705       return HRegionInfo.convert(response.getRegionInfo());
1706     } catch (ServiceException se) {
1707       throw getRemoteException(se);
1708     }
1709   }
1710 
1711   /**
1712    * A helper to close a region given a region name
1713    * using admin protocol.
1714    *
1715    * @param admin
1716    * @param regionName
1717    * @param transitionInZK
1718    * @throws IOException
1719    */
1720   public static void closeRegion(final RpcController controller, 
1721       final AdminService.BlockingInterface admin, final ServerName server, final byte[] regionName,
1722       final boolean transitionInZK) throws IOException {
1723     CloseRegionRequest closeRegionRequest =
1724       RequestConverter.buildCloseRegionRequest(server, regionName, transitionInZK);
1725     try {
1726       admin.closeRegion(controller, closeRegionRequest);
1727     } catch (ServiceException se) {
1728       throw getRemoteException(se);
1729     }
1730   }
1731 
1732   /**
1733    * A helper to close a region given a region name
1734    * using admin protocol.
1735    *
1736    * @param admin
1737    * @param regionName
1738    * @param versionOfClosingNode
1739    * @return true if the region is closed
1740    * @throws IOException
1741    */
1742   public static boolean closeRegion(final RpcController controller,
1743       final AdminService.BlockingInterface admin,
1744       final ServerName server,
1745       final byte[] regionName,
1746       final int versionOfClosingNode, final ServerName destinationServer,
1747       final boolean transitionInZK) throws IOException {
1748     CloseRegionRequest closeRegionRequest =
1749       RequestConverter.buildCloseRegionRequest(server,
1750         regionName, versionOfClosingNode, destinationServer, transitionInZK);
1751     try {
1752       CloseRegionResponse response = admin.closeRegion(controller, closeRegionRequest);
1753       return ResponseConverter.isClosed(response);
1754     } catch (ServiceException se) {
1755       throw getRemoteException(se);
1756     }
1757   }
1758 
1759   /**
1760    * A helper to warmup a region given a region name
1761    * using admin protocol
1762    *
1763    * @param admin
1764    * @param regionInfo
1765    *
1766    */
1767   public static void warmupRegion(final RpcController controller,
1768       final AdminService.BlockingInterface admin, final HRegionInfo regionInfo) throws IOException {
1769 
1770     try {
1771       WarmupRegionRequest warmupRegionRequest =
1772            RequestConverter.buildWarmupRegionRequest(regionInfo);
1773 
1774       admin.warmupRegion(controller, warmupRegionRequest);
1775     } catch (ServiceException e) {
1776       throw getRemoteException(e);
1777     }
1778   }
1779 
1780   /**
1781    * A helper to open a region using admin protocol.
1782    * @param admin
1783    * @param region
1784    * @throws IOException
1785    */
1786   public static void openRegion(final RpcController controller,
1787       final AdminService.BlockingInterface admin, ServerName server, final HRegionInfo region)
1788           throws IOException {
1789     OpenRegionRequest request =
1790       RequestConverter.buildOpenRegionRequest(server, region, -1, null, null);
1791     try {
1792       admin.openRegion(controller, request);
1793     } catch (ServiceException se) {
1794       throw ProtobufUtil.getRemoteException(se);
1795     }
1796   }
1797 
1798   /**
1799    * A helper to get the all the online regions on a region
1800    * server using admin protocol.
1801    *
1802    * @param admin
1803    * @return a list of online region info
1804    * @throws IOException
1805    */
1806   public static List<HRegionInfo> getOnlineRegions(final AdminService.BlockingInterface admin)
1807       throws IOException {
1808     return getOnlineRegions(null, admin);
1809   }
1810 
1811   /**
1812    * A helper to get the all the online regions on a region
1813    * server using admin protocol.
1814    * @return a list of online region info
1815    */
1816   public static List<HRegionInfo> getOnlineRegions(final RpcController controller,
1817       final AdminService.BlockingInterface admin)
1818   throws IOException {
1819     GetOnlineRegionRequest request = RequestConverter.buildGetOnlineRegionRequest();
1820     GetOnlineRegionResponse response = null;
1821     try {
1822       response = admin.getOnlineRegion(controller, request);
1823     } catch (ServiceException se) {
1824       throw getRemoteException(se);
1825     }
1826     return getRegionInfos(response);
1827   }
1828 
1829   /**
1830    * Get the list of region info from a GetOnlineRegionResponse
1831    *
1832    * @param proto the GetOnlineRegionResponse
1833    * @return the list of region info or null if <code>proto</code> is null
1834    */
1835   static List<HRegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
1836     if (proto == null) return null;
1837     List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
1838     for (RegionInfo regionInfo: proto.getRegionInfoList()) {
1839       regionInfos.add(HRegionInfo.convert(regionInfo));
1840     }
1841     return regionInfos;
1842   }
1843 
1844   /**
1845    * A helper to get the info of a region server using admin protocol.
1846    * @return the server name
1847    */
1848   public static ServerInfo getServerInfo(final RpcController controller,
1849       final AdminService.BlockingInterface admin)
1850   throws IOException {
1851     GetServerInfoRequest request = RequestConverter.buildGetServerInfoRequest();
1852     try {
1853       GetServerInfoResponse response = admin.getServerInfo(controller, request);
1854       return response.getServerInfo();
1855     } catch (ServiceException se) {
1856       throw getRemoteException(se);
1857     }
1858   }
1859 
1860   /**
1861    * A helper to get the list of files of a column family
1862    * on a given region using admin protocol.
1863    *
1864    * @return the list of store files
1865    */
1866   public static List<String> getStoreFiles(final AdminService.BlockingInterface admin,
1867       final byte[] regionName, final byte[] family)
1868   throws IOException {
1869     return getStoreFiles(null, admin, regionName, family);
1870   }
1871 
1872   /**
1873    * A helper to get the list of files of a column family
1874    * on a given region using admin protocol.
1875    *
1876    * @return the list of store files
1877    */
1878   public static List<String> getStoreFiles(final RpcController controller,
1879       final AdminService.BlockingInterface admin, final byte[] regionName, final byte[] family)
1880   throws IOException {
1881     GetStoreFileRequest request =
1882       RequestConverter.buildGetStoreFileRequest(regionName, family);
1883     try {
1884       GetStoreFileResponse response = admin.getStoreFile(controller, request);
1885       return response.getStoreFileList();
1886     } catch (ServiceException se) {
1887       throw ProtobufUtil.getRemoteException(se);
1888     }
1889   }
1890 
1891   /**
1892    * A helper to split a region using admin protocol.
1893    *
1894    * @param admin
1895    * @param hri
1896    * @param splitPoint
1897    * @throws IOException
1898    */
1899   public static void split(final RpcController controller,
1900       final AdminService.BlockingInterface admin, final HRegionInfo hri, byte[] splitPoint)
1901           throws IOException {
1902     SplitRegionRequest request =
1903       RequestConverter.buildSplitRegionRequest(hri.getRegionName(), splitPoint);
1904     try {
1905       admin.splitRegion(controller, request);
1906     } catch (ServiceException se) {
1907       throw ProtobufUtil.getRemoteException(se);
1908     }
1909   }
1910 
1911   /**
1912    * A helper to merge regions using admin protocol. Send request to
1913    * regionserver.
1914    * @param admin
1915    * @param region_a
1916    * @param region_b
1917    * @param forcible true if do a compulsory merge, otherwise we will only merge
1918    *          two adjacent regions
1919    * @throws IOException
1920    */
1921   public static void mergeRegions(final RpcController controller,
1922       final AdminService.BlockingInterface admin,
1923       final HRegionInfo region_a, final HRegionInfo region_b,
1924       final boolean forcible) throws IOException {
1925     MergeRegionsRequest request = RequestConverter.buildMergeRegionsRequest(
1926         region_a.getRegionName(), region_b.getRegionName(),forcible);
1927     try {
1928       admin.mergeRegions(controller, request);
1929     } catch (ServiceException se) {
1930       throw ProtobufUtil.getRemoteException(se);
1931     }
1932   }
1933 
1934 // End helpers for Admin
1935 
1936   /*
1937    * Get the total (read + write) requests from a RegionLoad pb
1938    * @param rl - RegionLoad pb
1939    * @return total (read + write) requests
1940    */
1941   public static long getTotalRequestsCount(RegionLoad rl) {
1942     if (rl == null) {
1943       return 0;
1944     }
1945 
1946     return rl.getReadRequestsCount() + rl.getWriteRequestsCount();
1947   }
1948 
1949 
1950   /**
1951    * @param m Message to get delimited pb serialization of (with pb magic prefix)
1952    */
1953   public static byte [] toDelimitedByteArray(final Message m) throws IOException {
1954     // Allocate arbitrary big size so we avoid resizing.
1955     ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
1956     baos.write(PB_MAGIC);
1957     m.writeDelimitedTo(baos);
1958     return baos.toByteArray();
1959   }
1960 
1961   /**
1962    * Converts a Permission proto to a client Permission object.
1963    *
1964    * @param proto the protobuf Permission
1965    * @return the converted Permission
1966    */
1967   public static Permission toPermission(AccessControlProtos.Permission proto) {
1968     if (proto.getType() != AccessControlProtos.Permission.Type.Global) {
1969       return toTablePermission(proto);
1970     } else {
1971       List<Permission.Action> actions = toPermissionActions(proto.getGlobalPermission().getActionList());
1972       return new Permission(actions.toArray(new Permission.Action[actions.size()]));
1973     }
1974   }
1975 
1976   /**
1977    * Converts a Permission proto to a client TablePermission object.
1978    *
1979    * @param proto the protobuf Permission
1980    * @return the converted TablePermission
1981    */
1982   public static TablePermission toTablePermission(AccessControlProtos.Permission proto) {
1983     if(proto.getType() == AccessControlProtos.Permission.Type.Global) {
1984       AccessControlProtos.GlobalPermission perm = proto.getGlobalPermission();
1985       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1986 
1987       return new TablePermission(null, null, null,
1988           actions.toArray(new Permission.Action[actions.size()]));
1989     }
1990     if(proto.getType() == AccessControlProtos.Permission.Type.Namespace) {
1991       AccessControlProtos.NamespacePermission perm = proto.getNamespacePermission();
1992       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1993 
1994       if(!proto.hasNamespacePermission()) {
1995         throw new IllegalStateException("Namespace must not be empty in NamespacePermission");
1996       }
1997       String namespace = perm.getNamespaceName().toStringUtf8();
1998       return new TablePermission(namespace, actions.toArray(new Permission.Action[actions.size()]));
1999     }
2000     if(proto.getType() == AccessControlProtos.Permission.Type.Table) {
2001       AccessControlProtos.TablePermission perm = proto.getTablePermission();
2002       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
2003 
2004       byte[] qualifier = null;
2005       byte[] family = null;
2006       TableName table = null;
2007 
2008       if (!perm.hasTableName()) {
2009         throw new IllegalStateException("TableName cannot be empty");
2010       }
2011       table = ProtobufUtil.toTableName(perm.getTableName());
2012 
2013       if (perm.hasFamily()) family = perm.getFamily().toByteArray();
2014       if (perm.hasQualifier()) qualifier = perm.getQualifier().toByteArray();
2015 
2016       return new TablePermission(table, family, qualifier,
2017           actions.toArray(new Permission.Action[actions.size()]));
2018     }
2019     throw new IllegalStateException("Unrecognize Perm Type: "+proto.getType());
2020   }
2021 
2022   /**
2023    * Convert a client Permission to a Permission proto
2024    *
2025    * @param perm the client Permission
2026    * @return the protobuf Permission
2027    */
2028   public static AccessControlProtos.Permission toPermission(Permission perm) {
2029     AccessControlProtos.Permission.Builder ret = AccessControlProtos.Permission.newBuilder();
2030     if (perm instanceof TablePermission) {
2031       TablePermission tablePerm = (TablePermission)perm;
2032       if(tablePerm.hasNamespace()) {
2033         ret.setType(AccessControlProtos.Permission.Type.Namespace);
2034 
2035         AccessControlProtos.NamespacePermission.Builder builder =
2036             AccessControlProtos.NamespacePermission.newBuilder();
2037         builder.setNamespaceName(ByteString.copyFromUtf8(tablePerm.getNamespace()));
2038         Permission.Action actions[] = perm.getActions();
2039         if (actions != null) {
2040           for (Permission.Action a : actions) {
2041             builder.addAction(toPermissionAction(a));
2042           }
2043         }
2044         ret.setNamespacePermission(builder);
2045         return ret.build();
2046       } else if (tablePerm.hasTable()) {
2047         ret.setType(AccessControlProtos.Permission.Type.Table);
2048 
2049         AccessControlProtos.TablePermission.Builder builder =
2050             AccessControlProtos.TablePermission.newBuilder();
2051         builder.setTableName(ProtobufUtil.toProtoTableName(tablePerm.getTableName()));
2052         if (tablePerm.hasFamily()) {
2053           builder.setFamily(ByteStringer.wrap(tablePerm.getFamily()));
2054         }
2055         if (tablePerm.hasQualifier()) {
2056           builder.setQualifier(ByteStringer.wrap(tablePerm.getQualifier()));
2057         }
2058         Permission.Action actions[] = perm.getActions();
2059         if (actions != null) {
2060           for (Permission.Action a : actions) {
2061             builder.addAction(toPermissionAction(a));
2062           }
2063         }
2064         ret.setTablePermission(builder);
2065         return ret.build();
2066       }
2067     }
2068 
2069     ret.setType(AccessControlProtos.Permission.Type.Global);
2070 
2071     AccessControlProtos.GlobalPermission.Builder builder =
2072         AccessControlProtos.GlobalPermission.newBuilder();
2073     Permission.Action actions[] = perm.getActions();
2074     if (actions != null) {
2075       for (Permission.Action a: actions) {
2076         builder.addAction(toPermissionAction(a));
2077       }
2078     }
2079     ret.setGlobalPermission(builder);
2080     return ret.build();
2081   }
2082 
2083   /**
2084    * Converts a list of Permission.Action proto to a list of client Permission.Action objects.
2085    *
2086    * @param protoActions the list of protobuf Actions
2087    * @return the converted list of Actions
2088    */
2089   public static List<Permission.Action> toPermissionActions(
2090       List<AccessControlProtos.Permission.Action> protoActions) {
2091     List<Permission.Action> actions = new ArrayList<Permission.Action>(protoActions.size());
2092     for (AccessControlProtos.Permission.Action a : protoActions) {
2093       actions.add(toPermissionAction(a));
2094     }
2095     return actions;
2096   }
2097 
2098   /**
2099    * Converts a Permission.Action proto to a client Permission.Action object.
2100    *
2101    * @param action the protobuf Action
2102    * @return the converted Action
2103    */
2104   public static Permission.Action toPermissionAction(
2105       AccessControlProtos.Permission.Action action) {
2106     switch (action) {
2107       case READ:
2108         return Permission.Action.READ;
2109       case WRITE:
2110         return Permission.Action.WRITE;
2111       case EXEC:
2112         return Permission.Action.EXEC;
2113       case CREATE:
2114         return Permission.Action.CREATE;
2115       case ADMIN:
2116         return Permission.Action.ADMIN;
2117     }
2118     throw new IllegalArgumentException("Unknown action value "+action.name());
2119   }
2120 
2121   /**
2122    * Convert a client Permission.Action to a Permission.Action proto
2123    *
2124    * @param action the client Action
2125    * @return the protobuf Action
2126    */
2127   public static AccessControlProtos.Permission.Action toPermissionAction(
2128       Permission.Action action) {
2129     switch (action) {
2130       case READ:
2131         return AccessControlProtos.Permission.Action.READ;
2132       case WRITE:
2133         return AccessControlProtos.Permission.Action.WRITE;
2134       case EXEC:
2135         return AccessControlProtos.Permission.Action.EXEC;
2136       case CREATE:
2137         return AccessControlProtos.Permission.Action.CREATE;
2138       case ADMIN:
2139         return AccessControlProtos.Permission.Action.ADMIN;
2140     }
2141     throw new IllegalArgumentException("Unknown action value "+action.name());
2142   }
2143 
2144   /**
2145    * Convert a client user permission to a user permission proto
2146    *
2147    * @param perm the client UserPermission
2148    * @return the protobuf UserPermission
2149    */
2150   public static AccessControlProtos.UserPermission toUserPermission(UserPermission perm) {
2151     return AccessControlProtos.UserPermission.newBuilder()
2152         .setUser(ByteStringer.wrap(perm.getUser()))
2153         .setPermission(toPermission(perm))
2154         .build();
2155   }
2156 
2157   /**
2158    * Converts a user permission proto to a client user permission object.
2159    *
2160    * @param proto the protobuf UserPermission
2161    * @return the converted UserPermission
2162    */
2163   public static UserPermission toUserPermission(AccessControlProtos.UserPermission proto) {
2164     return new UserPermission(proto.getUser().toByteArray(),
2165         toTablePermission(proto.getPermission()));
2166   }
2167 
2168   /**
2169    * Convert a ListMultimap&lt;String, TablePermission&gt; where key is username
2170    * to a protobuf UserPermission
2171    *
2172    * @param perm the list of user and table permissions
2173    * @return the protobuf UserTablePermissions
2174    */
2175   public static AccessControlProtos.UsersAndPermissions toUserTablePermissions(
2176       ListMultimap<String, TablePermission> perm) {
2177     AccessControlProtos.UsersAndPermissions.Builder builder =
2178                   AccessControlProtos.UsersAndPermissions.newBuilder();
2179     for (Map.Entry<String, Collection<TablePermission>> entry : perm.asMap().entrySet()) {
2180       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
2181                   AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
2182       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
2183       for (TablePermission tablePerm: entry.getValue()) {
2184         userPermBuilder.addPermissions(toPermission(tablePerm));
2185       }
2186       builder.addUserPermissions(userPermBuilder.build());
2187     }
2188     return builder.build();
2189   }
2190 
2191   /**
2192    * A utility used to grant a user global permissions.
2193    * <p>
2194    * It's also called by the shell, in case you want to find references.
2195    *
2196    * @param protocol the AccessControlService protocol proxy
2197    * @param userShortName the short name of the user to grant permissions
2198    * @param actions the permissions to be granted
2199    * @throws ServiceException
2200    */
2201   public static void grant(RpcController controller,
2202       AccessControlService.BlockingInterface protocol, String userShortName,
2203       Permission.Action... actions) throws ServiceException {
2204     List<AccessControlProtos.Permission.Action> permActions =
2205         Lists.newArrayListWithCapacity(actions.length);
2206     for (Permission.Action a : actions) {
2207       permActions.add(ProtobufUtil.toPermissionAction(a));
2208     }
2209     AccessControlProtos.GrantRequest request = RequestConverter.
2210       buildGrantRequest(userShortName, permActions.toArray(
2211         new AccessControlProtos.Permission.Action[actions.length]));
2212     protocol.grant(controller, request);
2213   }
2214 
2215   /**
2216    * A utility used to grant a user table permissions. The permissions will
2217    * be for a table table/column family/qualifier.
2218    * <p>
2219    * It's also called by the shell, in case you want to find references.
2220    *
2221    * @param protocol the AccessControlService protocol proxy
2222    * @param userShortName the short name of the user to grant permissions
2223    * @param tableName optional table name
2224    * @param f optional column family
2225    * @param q optional qualifier
2226    * @param actions the permissions to be granted
2227    * @throws ServiceException
2228    */
2229   public static void grant(RpcController controller,
2230       AccessControlService.BlockingInterface protocol, String userShortName, TableName tableName,
2231       byte[] f, byte[] q, Permission.Action... actions) throws ServiceException {
2232     List<AccessControlProtos.Permission.Action> permActions =
2233         Lists.newArrayListWithCapacity(actions.length);
2234     for (Permission.Action a : actions) {
2235       permActions.add(ProtobufUtil.toPermissionAction(a));
2236     }
2237     AccessControlProtos.GrantRequest request = RequestConverter.
2238       buildGrantRequest(userShortName, tableName, f, q, permActions.toArray(
2239         new AccessControlProtos.Permission.Action[actions.length]));
2240     protocol.grant(controller, request);
2241   }
2242 
2243   /**
2244    * A utility used to grant a user namespace permissions.
2245    * <p>
2246    * It's also called by the shell, in case you want to find references.
2247    *
2248    * @param protocol the AccessControlService protocol proxy
2249    * @param namespace the short name of the user to grant permissions
2250    * @param actions the permissions to be granted
2251    * @throws ServiceException
2252    */
2253   public static void grant(RpcController controller,
2254       AccessControlService.BlockingInterface protocol, String userShortName, String namespace,
2255       Permission.Action... actions) throws ServiceException {
2256     List<AccessControlProtos.Permission.Action> permActions =
2257         Lists.newArrayListWithCapacity(actions.length);
2258     for (Permission.Action a : actions) {
2259       permActions.add(ProtobufUtil.toPermissionAction(a));
2260     }
2261     AccessControlProtos.GrantRequest request = RequestConverter.
2262       buildGrantRequest(userShortName, namespace, permActions.toArray(
2263         new AccessControlProtos.Permission.Action[actions.length]));
2264     protocol.grant(controller, request);
2265   }
2266 
2267   /**
2268    * A utility used to revoke a user's global permissions.
2269    * <p>
2270    * It's also called by the shell, in case you want to find references.
2271    *
2272    * @param protocol the AccessControlService protocol proxy
2273    * @param userShortName the short name of the user to revoke permissions
2274    * @param actions the permissions to be revoked
2275    * @throws ServiceException
2276    */
2277   public static void revoke(RpcController controller,
2278       AccessControlService.BlockingInterface protocol, String userShortName,
2279       Permission.Action... actions) throws ServiceException {
2280     List<AccessControlProtos.Permission.Action> permActions =
2281         Lists.newArrayListWithCapacity(actions.length);
2282     for (Permission.Action a : actions) {
2283       permActions.add(ProtobufUtil.toPermissionAction(a));
2284     }
2285     AccessControlProtos.RevokeRequest request = RequestConverter.
2286       buildRevokeRequest(userShortName, permActions.toArray(
2287         new AccessControlProtos.Permission.Action[actions.length]));
2288     protocol.revoke(controller, request);
2289   }
2290 
2291   /**
2292    * A utility used to revoke a user's table permissions. The permissions will
2293    * be for a table/column family/qualifier.
2294    * <p>
2295    * It's also called by the shell, in case you want to find references.
2296    *
2297    * @param protocol the AccessControlService protocol proxy
2298    * @param userShortName the short name of the user to revoke permissions
2299    * @param tableName optional table name
2300    * @param f optional column family
2301    * @param q optional qualifier
2302    * @param actions the permissions to be revoked
2303    * @throws ServiceException
2304    */
2305   public static void revoke(RpcController controller,
2306       AccessControlService.BlockingInterface protocol, String userShortName, TableName tableName,
2307       byte[] f, byte[] q, Permission.Action... actions) throws ServiceException {
2308     List<AccessControlProtos.Permission.Action> permActions =
2309         Lists.newArrayListWithCapacity(actions.length);
2310     for (Permission.Action a : actions) {
2311       permActions.add(ProtobufUtil.toPermissionAction(a));
2312     }
2313     AccessControlProtos.RevokeRequest request = RequestConverter.
2314       buildRevokeRequest(userShortName, tableName, f, q, permActions.toArray(
2315         new AccessControlProtos.Permission.Action[actions.length]));
2316     protocol.revoke(controller, request);
2317   }
2318 
2319   /**
2320    * A utility used to revoke a user's namespace permissions.
2321    * <p>
2322    * It's also called by the shell, in case you want to find references.
2323    *
2324    * @param protocol the AccessControlService protocol proxy
2325    * @param userShortName the short name of the user to revoke permissions
2326    * @param namespace optional table name
2327    * @param actions the permissions to be revoked
2328    * @throws ServiceException
2329    */
2330   public static void revoke(RpcController controller,
2331       AccessControlService.BlockingInterface protocol, String userShortName, String namespace,
2332       Permission.Action... actions) throws ServiceException {
2333     List<AccessControlProtos.Permission.Action> permActions =
2334         Lists.newArrayListWithCapacity(actions.length);
2335     for (Permission.Action a : actions) {
2336       permActions.add(ProtobufUtil.toPermissionAction(a));
2337     }
2338     AccessControlProtos.RevokeRequest request = RequestConverter.
2339       buildRevokeRequest(userShortName, namespace, permActions.toArray(
2340         new AccessControlProtos.Permission.Action[actions.length]));
2341     protocol.revoke(controller, request);
2342   }
2343 
2344   /**
2345    * A utility used to get user's global permissions.
2346    * <p>
2347    * It's also called by the shell, in case you want to find references.
2348    *
2349    * @param protocol the AccessControlService protocol proxy
2350    * @throws ServiceException
2351    */
2352   public static List<UserPermission> getUserPermissions(RpcController controller,
2353       AccessControlService.BlockingInterface protocol) throws ServiceException {
2354     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2355       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2356     builder.setType(AccessControlProtos.Permission.Type.Global);
2357     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2358     AccessControlProtos.GetUserPermissionsResponse response =
2359       protocol.getUserPermissions(controller, request);
2360     List<UserPermission> perms = new ArrayList<UserPermission>(response.getUserPermissionCount());
2361     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2362       perms.add(ProtobufUtil.toUserPermission(perm));
2363     }
2364     return perms;
2365   }
2366 
2367   /**
2368    * A utility used to get user table permissions.
2369    * <p>
2370    * It's also called by the shell, in case you want to find references.
2371    *
2372    * @param protocol the AccessControlService protocol proxy
2373    * @param t optional table name
2374    * @throws ServiceException
2375    */
2376   public static List<UserPermission> getUserPermissions(RpcController controller,
2377       AccessControlService.BlockingInterface protocol,
2378       TableName t) throws ServiceException {
2379     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2380       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2381     if (t != null) {
2382       builder.setTableName(ProtobufUtil.toProtoTableName(t));
2383     }
2384     builder.setType(AccessControlProtos.Permission.Type.Table);
2385     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2386     AccessControlProtos.GetUserPermissionsResponse response =
2387       protocol.getUserPermissions(controller, request);
2388     List<UserPermission> perms = new ArrayList<UserPermission>(response.getUserPermissionCount());
2389     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2390       perms.add(ProtobufUtil.toUserPermission(perm));
2391     }
2392     return perms;
2393   }
2394 
2395   /**
2396    * A utility used to get permissions for selected namespace.
2397    * <p>
2398    * It's also called by the shell, in case you want to find references.
2399    *
2400    * @param protocol the AccessControlService protocol proxy
2401    * @param namespace name of the namespace
2402    * @throws ServiceException
2403    */
2404   public static List<UserPermission> getUserPermissions(RpcController controller,
2405       AccessControlService.BlockingInterface protocol,
2406       byte[] namespace) throws ServiceException {
2407     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2408       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2409     if (namespace != null) {
2410       builder.setNamespaceName(ByteStringer.wrap(namespace));
2411     }
2412     builder.setType(AccessControlProtos.Permission.Type.Namespace);
2413     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2414     AccessControlProtos.GetUserPermissionsResponse response =
2415       protocol.getUserPermissions(controller, request);
2416     List<UserPermission> perms = new ArrayList<UserPermission>(response.getUserPermissionCount());
2417     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2418       perms.add(ProtobufUtil.toUserPermission(perm));
2419     }
2420     return perms;
2421   }
2422 
2423   /**
2424    * Convert a protobuf UserTablePermissions to a
2425    * ListMultimap&lt;String, TablePermission&gt; where key is username.
2426    *
2427    * @param proto the protobuf UserPermission
2428    * @return the converted UserPermission
2429    */
2430   public static ListMultimap<String, TablePermission> toUserTablePermissions(
2431       AccessControlProtos.UsersAndPermissions proto) {
2432     ListMultimap<String, TablePermission> perms = ArrayListMultimap.create();
2433     AccessControlProtos.UsersAndPermissions.UserPermissions userPerm;
2434 
2435     for (int i = 0; i < proto.getUserPermissionsCount(); i++) {
2436       userPerm = proto.getUserPermissions(i);
2437       for (int j = 0; j < userPerm.getPermissionsCount(); j++) {
2438         TablePermission tablePerm = toTablePermission(userPerm.getPermissions(j));
2439         perms.put(userPerm.getUser().toStringUtf8(), tablePerm);
2440       }
2441     }
2442 
2443     return perms;
2444   }
2445 
2446   /**
2447    * Converts a Token instance (with embedded identifier) to the protobuf representation.
2448    *
2449    * @param token the Token instance to copy
2450    * @return the protobuf Token message
2451    */
2452   public static AuthenticationProtos.Token toToken(Token<AuthenticationTokenIdentifier> token) {
2453     AuthenticationProtos.Token.Builder builder = AuthenticationProtos.Token.newBuilder();
2454     builder.setIdentifier(ByteStringer.wrap(token.getIdentifier()));
2455     builder.setPassword(ByteStringer.wrap(token.getPassword()));
2456     if (token.getService() != null) {
2457       builder.setService(ByteString.copyFromUtf8(token.getService().toString()));
2458     }
2459     return builder.build();
2460   }
2461 
2462   /**
2463    * Converts a protobuf Token message back into a Token instance.
2464    *
2465    * @param proto the protobuf Token message
2466    * @return the Token instance
2467    */
2468   public static Token<AuthenticationTokenIdentifier> toToken(AuthenticationProtos.Token proto) {
2469     return new Token<AuthenticationTokenIdentifier>(
2470         proto.hasIdentifier() ? proto.getIdentifier().toByteArray() : null,
2471         proto.hasPassword() ? proto.getPassword().toByteArray() : null,
2472         AuthenticationTokenIdentifier.AUTH_TOKEN_TYPE,
2473         proto.hasService() ? new Text(proto.getService().toStringUtf8()) : null);
2474   }
2475 
2476   /**
2477    * Find the HRegion encoded name based on a region specifier
2478    *
2479    * @param regionSpecifier the region specifier
2480    * @return the corresponding region's encoded name
2481    * @throws DoNotRetryIOException if the specifier type is unsupported
2482    */
2483   public static String getRegionEncodedName(
2484       final RegionSpecifier regionSpecifier) throws DoNotRetryIOException {
2485     byte[] value = regionSpecifier.getValue().toByteArray();
2486     RegionSpecifierType type = regionSpecifier.getType();
2487     switch (type) {
2488       case REGION_NAME:
2489         return HRegionInfo.encodeRegionName(value);
2490       case ENCODED_REGION_NAME:
2491         return Bytes.toString(value);
2492       default:
2493         throw new DoNotRetryIOException(
2494           "Unsupported region specifier type: " + type);
2495     }
2496   }
2497 
2498   public static ScanMetrics toScanMetrics(final byte[] bytes) {
2499     Parser<MapReduceProtos.ScanMetrics> parser = MapReduceProtos.ScanMetrics.PARSER;
2500     MapReduceProtos.ScanMetrics pScanMetrics = null;
2501     try {
2502       pScanMetrics = parser.parseFrom(bytes);
2503     } catch (InvalidProtocolBufferException e) {
2504       //Ignored there are just no key values to add.
2505     }
2506     ScanMetrics scanMetrics = new ScanMetrics();
2507     if (pScanMetrics != null) {
2508       for (HBaseProtos.NameInt64Pair pair : pScanMetrics.getMetricsList()) {
2509         if (pair.hasName() && pair.hasValue()) {
2510           scanMetrics.setCounter(pair.getName(), pair.getValue());
2511         }
2512       }
2513     }
2514     return scanMetrics;
2515   }
2516 
2517   public static MapReduceProtos.ScanMetrics toScanMetrics(ScanMetrics scanMetrics) {
2518     MapReduceProtos.ScanMetrics.Builder builder = MapReduceProtos.ScanMetrics.newBuilder();
2519     Map<String, Long> metrics = scanMetrics.getMetricsMap();
2520     for (Entry<String, Long> e : metrics.entrySet()) {
2521       HBaseProtos.NameInt64Pair nameInt64Pair =
2522           HBaseProtos.NameInt64Pair.newBuilder()
2523               .setName(e.getKey())
2524               .setValue(e.getValue())
2525               .build();
2526       builder.addMetrics(nameInt64Pair);
2527     }
2528     return builder.build();
2529   }
2530 
2531   /**
2532    * Unwraps an exception from a protobuf service into the underlying (expected) IOException.
2533    * This method will <strong>always</strong> throw an exception.
2534    * @param se the {@code ServiceException} instance to convert into an {@code IOException}
2535    */
2536   public static void toIOException(ServiceException se) throws IOException {
2537     if (se == null) {
2538       throw new NullPointerException("Null service exception passed!");
2539     }
2540 
2541     Throwable cause = se.getCause();
2542     if (cause != null && cause instanceof IOException) {
2543       throw (IOException)cause;
2544     }
2545     throw new IOException(se);
2546   }
2547 
2548   public static CellProtos.Cell toCell(final Cell kv) {
2549     // Doing this is going to kill us if we do it for all data passed.
2550     // St.Ack 20121205
2551     CellProtos.Cell.Builder kvbuilder = CellProtos.Cell.newBuilder();
2552     kvbuilder.setRow(ByteStringer.wrap(kv.getRowArray(), kv.getRowOffset(),
2553         kv.getRowLength()));
2554     kvbuilder.setFamily(ByteStringer.wrap(kv.getFamilyArray(),
2555         kv.getFamilyOffset(), kv.getFamilyLength()));
2556     kvbuilder.setQualifier(ByteStringer.wrap(kv.getQualifierArray(),
2557         kv.getQualifierOffset(), kv.getQualifierLength()));
2558     kvbuilder.setCellType(CellProtos.CellType.valueOf(kv.getTypeByte()));
2559     kvbuilder.setTimestamp(kv.getTimestamp());
2560     kvbuilder.setValue(ByteStringer.wrap(kv.getValueArray(), kv.getValueOffset(),
2561         kv.getValueLength()));
2562     return kvbuilder.build();
2563   }
2564 
2565   public static Cell toCell(final CellProtos.Cell cell) {
2566     // Doing this is going to kill us if we do it for all data passed.
2567     // St.Ack 20121205
2568     return CellUtil.createCell(cell.getRow().toByteArray(),
2569       cell.getFamily().toByteArray(),
2570       cell.getQualifier().toByteArray(),
2571       cell.getTimestamp(),
2572       (byte)cell.getCellType().getNumber(),
2573       cell.getValue().toByteArray());
2574   }
2575 
2576   public static HBaseProtos.NamespaceDescriptor toProtoNamespaceDescriptor(NamespaceDescriptor ns) {
2577     HBaseProtos.NamespaceDescriptor.Builder b =
2578         HBaseProtos.NamespaceDescriptor.newBuilder()
2579             .setName(ByteString.copyFromUtf8(ns.getName()));
2580     for(Map.Entry<String, String> entry: ns.getConfiguration().entrySet()) {
2581       b.addConfiguration(HBaseProtos.NameStringPair.newBuilder()
2582           .setName(entry.getKey())
2583           .setValue(entry.getValue()));
2584     }
2585     return b.build();
2586   }
2587 
2588   public static NamespaceDescriptor toNamespaceDescriptor(
2589       HBaseProtos.NamespaceDescriptor desc) throws IOException {
2590     NamespaceDescriptor.Builder b =
2591       NamespaceDescriptor.create(desc.getName().toStringUtf8());
2592     for(HBaseProtos.NameStringPair prop : desc.getConfigurationList()) {
2593       b.addConfiguration(prop.getName(), prop.getValue());
2594     }
2595     return b.build();
2596   }
2597 
2598   /**
2599    * Get an instance of the argument type declared in a class's signature. The
2600    * argument type is assumed to be a PB Message subclass, and the instance is
2601    * created using parseFrom method on the passed ByteString.
2602    * @param runtimeClass the runtime type of the class
2603    * @param position the position of the argument in the class declaration
2604    * @param b the ByteString which should be parsed to get the instance created
2605    * @return the instance
2606    * @throws IOException
2607    */
2608   @SuppressWarnings("unchecked")
2609   public static <T extends Message>
2610   T getParsedGenericInstance(Class<?> runtimeClass, int position, ByteString b)
2611       throws IOException {
2612     Type type = runtimeClass.getGenericSuperclass();
2613     Type argType = ((ParameterizedType)type).getActualTypeArguments()[position];
2614     Class<T> classType = (Class<T>)argType;
2615     T inst;
2616     try {
2617       Method m = classType.getMethod("parseFrom", ByteString.class);
2618       inst = (T)m.invoke(null, b);
2619       return inst;
2620     } catch (SecurityException e) {
2621       throw new IOException(e);
2622     } catch (NoSuchMethodException e) {
2623       throw new IOException(e);
2624     } catch (IllegalArgumentException e) {
2625       throw new IOException(e);
2626     } catch (InvocationTargetException e) {
2627       throw new IOException(e);
2628     } catch (IllegalAccessException e) {
2629       throw new IOException(e);
2630     }
2631   }
2632 
2633   public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family,
2634       List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
2635     return toCompactionDescriptor(info, null, family, inputPaths, outputPaths, storeDir);
2636   }
2637 
2638   @SuppressWarnings("deprecation")
2639   public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] regionName,
2640       byte[] family, List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
2641     // compaction descriptor contains relative paths.
2642     // input / output paths are relative to the store dir
2643     // store dir is relative to region dir
2644     CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder()
2645         .setTableName(ByteStringer.wrap(info.getTableName()))
2646         .setEncodedRegionName(ByteStringer.wrap(
2647           regionName == null ? info.getEncodedNameAsBytes() : regionName))
2648         .setFamilyName(ByteStringer.wrap(family))
2649         .setStoreHomeDir(storeDir.getName()); //make relative
2650     for (Path inputPath : inputPaths) {
2651       builder.addCompactionInput(inputPath.getName()); //relative path
2652     }
2653     for (Path outputPath : outputPaths) {
2654       builder.addCompactionOutput(outputPath.getName());
2655     }
2656     builder.setRegionName(ByteStringer.wrap(info.getRegionName()));
2657     return builder.build();
2658   }
2659 
2660   public static FlushDescriptor toFlushDescriptor(FlushAction action, HRegionInfo hri,
2661       long flushSeqId, Map<byte[], List<Path>> committedFiles) {
2662     FlushDescriptor.Builder desc = FlushDescriptor.newBuilder()
2663         .setAction(action)
2664         .setEncodedRegionName(ByteStringer.wrap(hri.getEncodedNameAsBytes()))
2665         .setRegionName(ByteStringer.wrap(hri.getRegionName()))
2666         .setFlushSequenceNumber(flushSeqId)
2667         .setTableName(ByteStringer.wrap(hri.getTable().getName()));
2668 
2669     for (Map.Entry<byte[], List<Path>> entry : committedFiles.entrySet()) {
2670       WALProtos.FlushDescriptor.StoreFlushDescriptor.Builder builder =
2671           WALProtos.FlushDescriptor.StoreFlushDescriptor.newBuilder()
2672           .setFamilyName(ByteStringer.wrap(entry.getKey()))
2673           .setStoreHomeDir(Bytes.toString(entry.getKey())); //relative to region
2674       if (entry.getValue() != null) {
2675         for (Path path : entry.getValue()) {
2676           builder.addFlushOutput(path.getName());
2677         }
2678       }
2679       desc.addStoreFlushes(builder);
2680     }
2681     return desc.build();
2682   }
2683 
2684   public static RegionEventDescriptor toRegionEventDescriptor(
2685       EventType eventType, HRegionInfo hri, long seqId, ServerName server,
2686       Map<byte[], List<Path>> storeFiles) {
2687     final byte[] tableNameAsBytes = hri.getTable().getName();
2688     final byte[] encodedNameAsBytes = hri.getEncodedNameAsBytes();
2689     final byte[] regionNameAsBytes = hri.getRegionName();
2690     return toRegionEventDescriptor(eventType,
2691         tableNameAsBytes,
2692         encodedNameAsBytes,
2693         regionNameAsBytes,
2694         seqId,
2695 
2696         server,
2697         storeFiles);
2698   }
2699 
2700   public static RegionEventDescriptor toRegionEventDescriptor(EventType eventType,
2701                                                               byte[] tableNameAsBytes,
2702                                                               byte[] encodedNameAsBytes,
2703                                                               byte[] regionNameAsBytes,
2704                                                                long seqId,
2705 
2706                                                               ServerName server,
2707                                                               Map<byte[], List<Path>> storeFiles) {
2708     RegionEventDescriptor.Builder desc = RegionEventDescriptor.newBuilder()
2709         .setEventType(eventType)
2710         .setTableName(ByteStringer.wrap(tableNameAsBytes))
2711         .setEncodedRegionName(ByteStringer.wrap(encodedNameAsBytes))
2712         .setRegionName(ByteStringer.wrap(regionNameAsBytes))
2713         .setLogSequenceNumber(seqId)
2714         .setServer(toServerName(server));
2715 
2716     for (Entry<byte[], List<Path>> entry : storeFiles.entrySet()) {
2717       StoreDescriptor.Builder builder = StoreDescriptor.newBuilder()
2718           .setFamilyName(ByteStringer.wrap(entry.getKey()))
2719           .setStoreHomeDir(Bytes.toString(entry.getKey()));
2720       for (Path path : entry.getValue()) {
2721         builder.addStoreFile(path.getName());
2722       }
2723 
2724       desc.addStores(builder);
2725     }
2726     return desc.build();
2727   }
2728 
2729   /**
2730    * Return short version of Message toString'd, shorter than TextFormat#shortDebugString.
2731    * Tries to NOT print out data both because it can be big but also so we do not have data in our
2732    * logs. Use judiciously.
2733    * @param m
2734    * @return toString of passed <code>m</code>
2735    */
2736   public static String getShortTextFormat(Message m) {
2737     if (m == null) return "null";
2738     if (m instanceof ScanRequest) {
2739       // This should be small and safe to output.  No data.
2740       return TextFormat.shortDebugString(m);
2741     } else if (m instanceof RegionServerReportRequest) {
2742       // Print a short message only, just the servername and the requests, not the full load.
2743       RegionServerReportRequest r = (RegionServerReportRequest)m;
2744       return "server " + TextFormat.shortDebugString(r.getServer()) +
2745         " load { numberOfRequests: " + r.getLoad().getNumberOfRequests() + " }";
2746     } else if (m instanceof RegionServerStartupRequest) {
2747       // Should be small enough.
2748       return TextFormat.shortDebugString(m);
2749     } else if (m instanceof MutationProto) {
2750       return toShortString((MutationProto)m);
2751     } else if (m instanceof GetRequest) {
2752       GetRequest r = (GetRequest) m;
2753       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2754           ", row=" + getStringForByteString(r.getGet().getRow());
2755     } else if (m instanceof ClientProtos.MultiRequest) {
2756       ClientProtos.MultiRequest r = (ClientProtos.MultiRequest) m;
2757       // Get first set of Actions.
2758       ClientProtos.RegionAction actions = r.getRegionActionList().get(0);
2759       String row = actions.getActionCount() <= 0? "":
2760         getStringForByteString(actions.getAction(0).hasGet()?
2761           actions.getAction(0).getGet().getRow():
2762           actions.getAction(0).getMutation().getRow());
2763       return "region= " + getStringForByteString(actions.getRegion().getValue()) +
2764           ", for " + r.getRegionActionCount() +
2765           " actions and 1st row key=" + row;
2766     } else if (m instanceof ClientProtos.MutateRequest) {
2767       ClientProtos.MutateRequest r = (ClientProtos.MutateRequest) m;
2768       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2769           ", row=" + getStringForByteString(r.getMutation().getRow());
2770     }
2771     return "TODO: " + m.getClass().toString();
2772   }
2773 
2774   private static String getStringForByteString(ByteString bs) {
2775     return Bytes.toStringBinary(bs.toByteArray());
2776   }
2777 
2778   /**
2779    * Print out some subset of a MutationProto rather than all of it and its data
2780    * @param proto Protobuf to print out
2781    * @return Short String of mutation proto
2782    */
2783   static String toShortString(final MutationProto proto) {
2784     return "row=" + Bytes.toString(proto.getRow().toByteArray()) +
2785         ", type=" + proto.getMutateType().toString();
2786   }
2787 
2788   public static TableName toTableName(HBaseProtos.TableName tableNamePB) {
2789     return TableName.valueOf(tableNamePB.getNamespace().asReadOnlyByteBuffer(),
2790         tableNamePB.getQualifier().asReadOnlyByteBuffer());
2791   }
2792 
2793   public static HBaseProtos.TableName toProtoTableName(TableName tableName) {
2794     return HBaseProtos.TableName.newBuilder()
2795         .setNamespace(ByteStringer.wrap(tableName.getNamespace()))
2796         .setQualifier(ByteStringer.wrap(tableName.getQualifier())).build();
2797   }
2798 
2799   public static TableName[] getTableNameArray(List<HBaseProtos.TableName> tableNamesList) {
2800     if (tableNamesList == null) {
2801       return new TableName[0];
2802     }
2803     TableName[] tableNames = new TableName[tableNamesList.size()];
2804     for (int i = 0; i < tableNamesList.size(); i++) {
2805       tableNames[i] = toTableName(tableNamesList.get(i));
2806     }
2807     return tableNames;
2808   }
2809 
2810   /**
2811    * Convert a protocol buffer CellVisibility to a client CellVisibility
2812    *
2813    * @param proto
2814    * @return the converted client CellVisibility
2815    */
2816   public static CellVisibility toCellVisibility(ClientProtos.CellVisibility proto) {
2817     if (proto == null) return null;
2818     return new CellVisibility(proto.getExpression());
2819   }
2820 
2821   /**
2822    * Convert a protocol buffer CellVisibility bytes to a client CellVisibility
2823    *
2824    * @param protoBytes
2825    * @return the converted client CellVisibility
2826    * @throws DeserializationException
2827    */
2828   public static CellVisibility toCellVisibility(byte[] protoBytes) throws DeserializationException {
2829     if (protoBytes == null) return null;
2830     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2831     ClientProtos.CellVisibility proto = null;
2832     try {
2833       ProtobufUtil.mergeFrom(builder, protoBytes);
2834       proto = builder.build();
2835     } catch (IOException e) {
2836       throw new DeserializationException(e);
2837     }
2838     return toCellVisibility(proto);
2839   }
2840 
2841   /**
2842    * Create a protocol buffer CellVisibility based on a client CellVisibility.
2843    *
2844    * @param cellVisibility
2845    * @return a protocol buffer CellVisibility
2846    */
2847   public static ClientProtos.CellVisibility toCellVisibility(CellVisibility cellVisibility) {
2848     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2849     builder.setExpression(cellVisibility.getExpression());
2850     return builder.build();
2851   }
2852 
2853   /**
2854    * Convert a protocol buffer Authorizations to a client Authorizations
2855    *
2856    * @param proto
2857    * @return the converted client Authorizations
2858    */
2859   public static Authorizations toAuthorizations(ClientProtos.Authorizations proto) {
2860     if (proto == null) return null;
2861     return new Authorizations(proto.getLabelList());
2862   }
2863 
2864   /**
2865    * Convert a protocol buffer Authorizations bytes to a client Authorizations
2866    *
2867    * @param protoBytes
2868    * @return the converted client Authorizations
2869    * @throws DeserializationException
2870    */
2871   public static Authorizations toAuthorizations(byte[] protoBytes) throws DeserializationException {
2872     if (protoBytes == null) return null;
2873     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2874     ClientProtos.Authorizations proto = null;
2875     try {
2876       ProtobufUtil.mergeFrom(builder, protoBytes);
2877       proto = builder.build();
2878     } catch (IOException e) {
2879       throw new DeserializationException(e);
2880     }
2881     return toAuthorizations(proto);
2882   }
2883 
2884   /**
2885    * Create a protocol buffer Authorizations based on a client Authorizations.
2886    *
2887    * @param authorizations
2888    * @return a protocol buffer Authorizations
2889    */
2890   public static ClientProtos.Authorizations toAuthorizations(Authorizations authorizations) {
2891     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2892     for (String label : authorizations.getLabels()) {
2893       builder.addLabel(label);
2894     }
2895     return builder.build();
2896   }
2897 
2898   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(String user,
2899       Permission perms) {
2900     return AccessControlProtos.UsersAndPermissions.newBuilder()
2901       .addUserPermissions(AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder()
2902         .setUser(ByteString.copyFromUtf8(user))
2903         .addPermissions(toPermission(perms))
2904         .build())
2905       .build();
2906   }
2907 
2908   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(
2909       ListMultimap<String, Permission> perms) {
2910     AccessControlProtos.UsersAndPermissions.Builder builder =
2911         AccessControlProtos.UsersAndPermissions.newBuilder();
2912     for (Map.Entry<String, Collection<Permission>> entry : perms.asMap().entrySet()) {
2913       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
2914         AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
2915       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
2916       for (Permission perm: entry.getValue()) {
2917         userPermBuilder.addPermissions(toPermission(perm));
2918       }
2919       builder.addUserPermissions(userPermBuilder.build());
2920     }
2921     return builder.build();
2922   }
2923 
2924   public static ListMultimap<String, Permission> toUsersAndPermissions(
2925       AccessControlProtos.UsersAndPermissions proto) {
2926     ListMultimap<String, Permission> result = ArrayListMultimap.create();
2927     for (AccessControlProtos.UsersAndPermissions.UserPermissions userPerms:
2928         proto.getUserPermissionsList()) {
2929       String user = userPerms.getUser().toStringUtf8();
2930       for (AccessControlProtos.Permission perm: userPerms.getPermissionsList()) {
2931         result.put(user, toPermission(perm));
2932       }
2933     }
2934     return result;
2935   }
2936   
2937   /**
2938    * Convert a protocol buffer TimeUnit to a client TimeUnit
2939    * @param proto
2940    * @return the converted client TimeUnit
2941    */
2942   public static TimeUnit toTimeUnit(final HBaseProtos.TimeUnit proto) {
2943     switch (proto) {
2944     case NANOSECONDS:
2945       return TimeUnit.NANOSECONDS;
2946     case MICROSECONDS:
2947       return TimeUnit.MICROSECONDS;
2948     case MILLISECONDS:
2949       return TimeUnit.MILLISECONDS;
2950     case SECONDS:
2951       return TimeUnit.SECONDS;
2952     case MINUTES:
2953       return TimeUnit.MINUTES;
2954     case HOURS:
2955       return TimeUnit.HOURS;
2956     case DAYS:
2957       return TimeUnit.DAYS;
2958     default:
2959       throw new RuntimeException("Invalid TimeUnit " + proto);
2960     }
2961   }
2962 
2963   /**
2964    * Convert a client TimeUnit to a protocol buffer TimeUnit
2965    * @param timeUnit
2966    * @return the converted protocol buffer TimeUnit
2967    */
2968   public static HBaseProtos.TimeUnit toProtoTimeUnit(final TimeUnit timeUnit) {
2969     switch (timeUnit) {
2970     case NANOSECONDS:
2971       return HBaseProtos.TimeUnit.NANOSECONDS;
2972     case MICROSECONDS:
2973       return HBaseProtos.TimeUnit.MICROSECONDS;
2974     case MILLISECONDS:
2975       return HBaseProtos.TimeUnit.MILLISECONDS;
2976     case SECONDS:
2977       return HBaseProtos.TimeUnit.SECONDS;
2978     case MINUTES:
2979       return HBaseProtos.TimeUnit.MINUTES;
2980     case HOURS:
2981       return HBaseProtos.TimeUnit.HOURS;
2982     case DAYS:
2983       return HBaseProtos.TimeUnit.DAYS;
2984     default:
2985       throw new RuntimeException("Invalid TimeUnit " + timeUnit);
2986     }
2987   }
2988 
2989   /**
2990    * Convert a protocol buffer ThrottleType to a client ThrottleType
2991    * @param proto
2992    * @return the converted client ThrottleType
2993    */
2994   public static ThrottleType toThrottleType(final QuotaProtos.ThrottleType proto) {
2995     switch (proto) {
2996     case REQUEST_NUMBER:
2997       return ThrottleType.REQUEST_NUMBER;
2998     case REQUEST_SIZE:
2999       return ThrottleType.REQUEST_SIZE;
3000     case WRITE_NUMBER:
3001       return ThrottleType.WRITE_NUMBER;
3002     case WRITE_SIZE:
3003       return ThrottleType.WRITE_SIZE;
3004     case READ_NUMBER:
3005       return ThrottleType.READ_NUMBER;
3006     case READ_SIZE:
3007       return ThrottleType.READ_SIZE;
3008     default:
3009       throw new RuntimeException("Invalid ThrottleType " + proto);
3010     }
3011   }
3012 
3013   /**
3014    * Convert a client ThrottleType to a protocol buffer ThrottleType
3015    * @param type
3016    * @return the converted protocol buffer ThrottleType
3017    */
3018   public static QuotaProtos.ThrottleType toProtoThrottleType(final ThrottleType type) {
3019     switch (type) {
3020     case REQUEST_NUMBER:
3021       return QuotaProtos.ThrottleType.REQUEST_NUMBER;
3022     case REQUEST_SIZE:
3023       return QuotaProtos.ThrottleType.REQUEST_SIZE;
3024     case WRITE_NUMBER:
3025       return QuotaProtos.ThrottleType.WRITE_NUMBER;
3026     case WRITE_SIZE:
3027       return QuotaProtos.ThrottleType.WRITE_SIZE;
3028     case READ_NUMBER:
3029       return QuotaProtos.ThrottleType.READ_NUMBER;
3030     case READ_SIZE:
3031       return QuotaProtos.ThrottleType.READ_SIZE;
3032     default:
3033       throw new RuntimeException("Invalid ThrottleType " + type);
3034     }
3035   }
3036 
3037   /**
3038    * Convert a protocol buffer QuotaScope to a client QuotaScope
3039    * @param proto
3040    * @return the converted client QuotaScope
3041    */
3042   public static QuotaScope toQuotaScope(final QuotaProtos.QuotaScope proto) {
3043     switch (proto) {
3044     case CLUSTER:
3045       return QuotaScope.CLUSTER;
3046     case MACHINE:
3047       return QuotaScope.MACHINE;
3048     default:
3049       throw new RuntimeException("Invalid QuotaScope " + proto);
3050     }
3051   }
3052 
3053   /**
3054    * Convert a client QuotaScope to a protocol buffer QuotaScope
3055    * @param scope
3056    * @return the converted protocol buffer QuotaScope
3057    */
3058   public static QuotaProtos.QuotaScope toProtoQuotaScope(final QuotaScope scope) {
3059     switch (scope) {
3060     case CLUSTER:
3061       return QuotaProtos.QuotaScope.CLUSTER;
3062     case MACHINE:
3063       return QuotaProtos.QuotaScope.MACHINE;
3064     default:
3065       throw new RuntimeException("Invalid QuotaScope " + scope);
3066     }
3067   }
3068 
3069   /**
3070    * Convert a protocol buffer QuotaType to a client QuotaType
3071    * @param proto
3072    * @return the converted client QuotaType
3073    */
3074   public static QuotaType toQuotaScope(final QuotaProtos.QuotaType proto) {
3075     switch (proto) {
3076     case THROTTLE:
3077       return QuotaType.THROTTLE;
3078     default:
3079       throw new RuntimeException("Invalid QuotaType " + proto);
3080     }
3081   }
3082 
3083   /**
3084    * Convert a client QuotaType to a protocol buffer QuotaType
3085    * @param type
3086    * @return the converted protocol buffer QuotaType
3087    */
3088   public static QuotaProtos.QuotaType toProtoQuotaScope(final QuotaType type) {
3089     switch (type) {
3090     case THROTTLE:
3091       return QuotaProtos.QuotaType.THROTTLE;
3092     default:
3093       throw new RuntimeException("Invalid QuotaType " + type);
3094     }
3095   }
3096 
3097   /**
3098    * Build a protocol buffer TimedQuota
3099    * @param limit the allowed number of request/data per timeUnit
3100    * @param timeUnit the limit time unit
3101    * @param scope the quota scope
3102    * @return the protocol buffer TimedQuota
3103    */
3104   public static QuotaProtos.TimedQuota toTimedQuota(final long limit, final TimeUnit timeUnit,
3105       final QuotaScope scope) {
3106     return QuotaProtos.TimedQuota.newBuilder().setSoftLimit(limit)
3107         .setTimeUnit(toProtoTimeUnit(timeUnit)).setScope(toProtoQuotaScope(scope)).build();
3108   }
3109 
3110   /**
3111    * Generates a marker for the WAL so that we propagate the notion of a bulk region load
3112    * throughout the WAL.
3113    *
3114    * @param tableName         The tableName into which the bulk load is being imported into.
3115    * @param encodedRegionName Encoded region name of the region which is being bulk loaded.
3116    * @param storeFiles        A set of store files of a column family are bulk loaded.
3117    * @param bulkloadSeqId     sequence ID (by a force flush) used to create bulk load hfile
3118    *                          name
3119    * @return The WAL log marker for bulk loads.
3120    */
3121   public static WALProtos.BulkLoadDescriptor toBulkLoadDescriptor(TableName tableName,
3122       ByteString encodedRegionName, Map<byte[], List<Path>> storeFiles, long bulkloadSeqId) {
3123     BulkLoadDescriptor.Builder desc = BulkLoadDescriptor.newBuilder()
3124         .setTableName(ProtobufUtil.toProtoTableName(tableName))
3125         .setEncodedRegionName(encodedRegionName).setBulkloadSeqNum(bulkloadSeqId);
3126 
3127     for (Map.Entry<byte[], List<Path>> entry : storeFiles.entrySet()) {
3128       WALProtos.StoreDescriptor.Builder builder = StoreDescriptor.newBuilder()
3129           .setFamilyName(ByteStringer.wrap(entry.getKey()))
3130           .setStoreHomeDir(Bytes.toString(entry.getKey())); // relative to region
3131       for (Path path : entry.getValue()) {
3132         builder.addStoreFile(path.getName());
3133       }
3134       desc.addStores(builder);
3135     }
3136 
3137     return desc.build();
3138   }
3139 
3140   /**
3141    * This version of protobuf's mergeDelimitedFrom avoids the hard-coded 64MB limit for decoding
3142    * buffers
3143    * @param builder current message builder
3144    * @param in Inputsream with delimited protobuf data
3145    * @throws IOException
3146    */
3147   public static void mergeDelimitedFrom(Message.Builder builder, InputStream in)
3148     throws IOException {
3149     // This used to be builder.mergeDelimitedFrom(in);
3150     // but is replaced to allow us to bump the protobuf size limit.
3151     final int firstByte = in.read();
3152     if (firstByte != -1) {
3153       final int size = CodedInputStream.readRawVarint32(firstByte, in);
3154       final InputStream limitedInput = new LimitInputStream(in, size);
3155       final CodedInputStream codedInput = CodedInputStream.newInstance(limitedInput);
3156       codedInput.setSizeLimit(size);
3157       builder.mergeFrom(codedInput);
3158       codedInput.checkLastTagWas(0);
3159     }
3160   }
3161 
3162   /**
3163    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3164    * buffers where the message size is known
3165    * @param builder current message builder
3166    * @param in InputStream containing protobuf data
3167    * @param size known size of protobuf data
3168    * @throws IOException 
3169    */
3170   public static void mergeFrom(Message.Builder builder, InputStream in, int size)
3171       throws IOException {
3172     final CodedInputStream codedInput = CodedInputStream.newInstance(in);
3173     codedInput.setSizeLimit(size);
3174     builder.mergeFrom(codedInput);
3175     codedInput.checkLastTagWas(0);
3176   }
3177 
3178   /**
3179    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3180    * buffers where the message size is not known
3181    * @param builder current message builder
3182    * @param in InputStream containing protobuf data
3183    * @throws IOException 
3184    */
3185   public static void mergeFrom(Message.Builder builder, InputStream in)
3186       throws IOException {
3187     final CodedInputStream codedInput = CodedInputStream.newInstance(in);
3188     codedInput.setSizeLimit(Integer.MAX_VALUE);
3189     builder.mergeFrom(codedInput);
3190     codedInput.checkLastTagWas(0);
3191   }
3192 
3193   /**
3194    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3195    * buffers when working with ByteStrings
3196    * @param builder current message builder
3197    * @param bs ByteString containing the 
3198    * @throws IOException 
3199    */
3200   public static void mergeFrom(Message.Builder builder, ByteString bs) throws IOException {
3201     final CodedInputStream codedInput = bs.newCodedInput();
3202     codedInput.setSizeLimit(bs.size());
3203     builder.mergeFrom(codedInput);
3204     codedInput.checkLastTagWas(0);
3205   }
3206 
3207   /**
3208    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3209    * buffers when working with byte arrays
3210    * @param builder current message builder
3211    * @param b byte array
3212    * @throws IOException 
3213    */
3214   public static void mergeFrom(Message.Builder builder, byte[] b) throws IOException {
3215     final CodedInputStream codedInput = CodedInputStream.newInstance(b);
3216     codedInput.setSizeLimit(b.length);
3217     builder.mergeFrom(codedInput);
3218     codedInput.checkLastTagWas(0);
3219   }
3220 
3221   /**
3222    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3223    * buffers when working with byte arrays
3224    * @param builder current message builder
3225    * @param b byte array
3226    * @param offset
3227    * @param length
3228    * @throws IOException
3229    */
3230   public static void mergeFrom(Message.Builder builder, byte[] b, int offset, int length)
3231       throws IOException {
3232     final CodedInputStream codedInput = CodedInputStream.newInstance(b, offset, length);
3233     codedInput.setSizeLimit(length);
3234     builder.mergeFrom(codedInput);
3235     codedInput.checkLastTagWas(0);
3236   }
3237 
3238   public static ReplicationLoadSink toReplicationLoadSink(
3239       ClusterStatusProtos.ReplicationLoadSink cls) {
3240     return new ReplicationLoadSink(cls.getAgeOfLastAppliedOp(), cls.getTimeStampsOfLastAppliedOp());
3241   }
3242 
3243   public static ReplicationLoadSource toReplicationLoadSource(
3244       ClusterStatusProtos.ReplicationLoadSource cls) {
3245     return new ReplicationLoadSource(cls.getPeerID(), cls.getAgeOfLastShippedOp(),
3246         cls.getSizeOfLogQueue(), cls.getTimeStampOfLastShippedOp(), cls.getReplicationLag());
3247   }
3248 
3249   public static List<ReplicationLoadSource> toReplicationLoadSourceList(
3250       List<ClusterStatusProtos.ReplicationLoadSource> clsList) {
3251     ArrayList<ReplicationLoadSource> rlsList = new ArrayList<ReplicationLoadSource>();
3252     for (ClusterStatusProtos.ReplicationLoadSource cls : clsList) {
3253       rlsList.add(toReplicationLoadSource(cls));
3254     }
3255     return rlsList;
3256   }
3257 
3258   /**
3259    * Get a protocol buffer VersionInfo
3260    *
3261    * @return the converted protocol buffer VersionInfo
3262    */
3263   public static HBaseProtos.VersionInfo getVersionInfo() {
3264     HBaseProtos.VersionInfo.Builder builder = HBaseProtos.VersionInfo.newBuilder();
3265     builder.setVersion(VersionInfo.getVersion());
3266     builder.setUrl(VersionInfo.getUrl());
3267     builder.setRevision(VersionInfo.getRevision());
3268     builder.setUser(VersionInfo.getUser());
3269     builder.setDate(VersionInfo.getDate());
3270     builder.setSrcChecksum(VersionInfo.getSrcChecksum());
3271     return builder.build();
3272   }
3273 
3274   /**
3275    * Convert SecurityCapabilitiesResponse.Capability to SecurityCapability
3276    * @param capabilities capabilities returned in the SecurityCapabilitiesResponse message
3277    * @return the converted list of SecurityCapability elements
3278    */
3279   public static List<SecurityCapability> toSecurityCapabilityList(
3280       List<MasterProtos.SecurityCapabilitiesResponse.Capability> capabilities) {
3281     List<SecurityCapability> scList = new ArrayList<>(capabilities.size());
3282     for (MasterProtos.SecurityCapabilitiesResponse.Capability c: capabilities) {
3283       try {
3284         scList.add(SecurityCapability.valueOf(c.getNumber()));
3285       } catch (IllegalArgumentException e) {
3286         // Unknown capability, just ignore it. We don't understand the new capability
3287         // but don't care since by definition we cannot take advantage of it.
3288       }
3289     }
3290     return scList;
3291   }
3292 
3293   private static HBaseProtos.TimeRange.Builder timeRangeToProto(TimeRange timeRange) {
3294     HBaseProtos.TimeRange.Builder timeRangeBuilder =
3295         HBaseProtos.TimeRange.newBuilder();
3296     timeRangeBuilder.setFrom(timeRange.getMin());
3297     timeRangeBuilder.setTo(timeRange.getMax());
3298     return timeRangeBuilder;
3299   }
3300 
3301   private static TimeRange protoToTimeRange(HBaseProtos.TimeRange timeRange) throws IOException {
3302       long minStamp = 0;
3303       long maxStamp = Long.MAX_VALUE;
3304       if (timeRange.hasFrom()) {
3305         minStamp = timeRange.getFrom();
3306       }
3307       if (timeRange.hasTo()) {
3308         maxStamp = timeRange.getTo();
3309       }
3310     return new TimeRange(minStamp, maxStamp);
3311   }
3312 
3313 }