View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.protobuf;
19  
20  
21  import static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME;
22  
23  import java.io.ByteArrayOutputStream;
24  import java.io.IOException;
25  import java.io.InputStream;
26  import java.io.InterruptedIOException;
27  import java.lang.reflect.Constructor;
28  import java.lang.reflect.InvocationTargetException;
29  import java.lang.reflect.Method;
30  import java.lang.reflect.ParameterizedType;
31  import java.lang.reflect.Type;
32  import java.nio.ByteBuffer;
33  import java.security.PrivilegedExceptionAction;
34  import java.util.ArrayList;
35  import java.util.Collection;
36  import java.util.HashMap;
37  import java.util.List;
38  import java.util.Map;
39  import java.util.Map.Entry;
40  import java.util.NavigableSet;
41  import java.util.concurrent.TimeUnit;
42  
43  import org.apache.hadoop.conf.Configuration;
44  import org.apache.hadoop.fs.Path;
45  import org.apache.hadoop.hbase.Cell;
46  import org.apache.hadoop.hbase.CellScanner;
47  import org.apache.hadoop.hbase.CellUtil;
48  import org.apache.hadoop.hbase.DoNotRetryIOException;
49  import org.apache.hadoop.hbase.HBaseConfiguration;
50  import org.apache.hadoop.hbase.HConstants;
51  import org.apache.hadoop.hbase.HRegionInfo;
52  import org.apache.hadoop.hbase.HTableDescriptor;
53  import org.apache.hadoop.hbase.KeyValue;
54  import org.apache.hadoop.hbase.NamespaceDescriptor;
55  import org.apache.hadoop.hbase.ServerName;
56  import org.apache.hadoop.hbase.TableName;
57  import org.apache.hadoop.hbase.Tag;
58  import org.apache.hadoop.hbase.TagUtil;
59  import org.apache.hadoop.hbase.classification.InterfaceAudience;
60  import org.apache.hadoop.hbase.client.Append;
61  import org.apache.hadoop.hbase.client.Consistency;
62  import org.apache.hadoop.hbase.client.Delete;
63  import org.apache.hadoop.hbase.client.Durability;
64  import org.apache.hadoop.hbase.client.Get;
65  import org.apache.hadoop.hbase.client.Increment;
66  import org.apache.hadoop.hbase.client.Mutation;
67  import org.apache.hadoop.hbase.client.Put;
68  import org.apache.hadoop.hbase.client.Result;
69  import org.apache.hadoop.hbase.client.Scan;
70  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
71  import org.apache.hadoop.hbase.client.security.SecurityCapability;
72  import org.apache.hadoop.hbase.exceptions.DeserializationException;
73  import org.apache.hadoop.hbase.filter.ByteArrayComparable;
74  import org.apache.hadoop.hbase.filter.Filter;
75  import org.apache.hadoop.hbase.io.LimitInputStream;
76  import org.apache.hadoop.hbase.io.TimeRange;
77  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
78  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService;
79  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
80  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
81  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
82  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
83  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
84  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
85  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
86  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
87  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
88  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
89  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
90  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
91  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
92  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
93  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
94  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest;
95  import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
96  import org.apache.hadoop.hbase.protobuf.generated.CellProtos;
97  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
98  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
99  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
100 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
101 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Column;
102 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall;
103 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
104 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
105 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
106 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
107 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue;
108 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue.QualifierValue;
109 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.DeleteType;
110 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
111 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
112 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
113 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
114 import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
115 import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
116 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
117 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
118 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo;
119 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
120 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
121 import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
122 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos;
123 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
124 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
125 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService;
126 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos;
127 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
128 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
129 import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
130 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.BulkLoadDescriptor;
131 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
132 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
133 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
134 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
135 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor.EventType;
136 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
137 import org.apache.hadoop.hbase.quotas.QuotaScope;
138 import org.apache.hadoop.hbase.quotas.QuotaType;
139 import org.apache.hadoop.hbase.quotas.ThrottleType;
140 import org.apache.hadoop.hbase.replication.ReplicationLoadSink;
141 import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
142 import org.apache.hadoop.hbase.security.User;
143 import org.apache.hadoop.hbase.security.access.Permission;
144 import org.apache.hadoop.hbase.security.access.TablePermission;
145 import org.apache.hadoop.hbase.security.access.UserPermission;
146 import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier;
147 import org.apache.hadoop.hbase.security.visibility.Authorizations;
148 import org.apache.hadoop.hbase.security.visibility.CellVisibility;
149 import org.apache.hadoop.hbase.util.ByteStringer;
150 import org.apache.hadoop.hbase.util.Bytes;
151 import org.apache.hadoop.hbase.util.DynamicClassLoader;
152 import org.apache.hadoop.hbase.util.ExceptionUtil;
153 import org.apache.hadoop.hbase.util.Methods;
154 import org.apache.hadoop.hbase.util.Pair;
155 import org.apache.hadoop.hbase.util.VersionInfo;
156 import org.apache.hadoop.io.Text;
157 import org.apache.hadoop.ipc.RemoteException;
158 import org.apache.hadoop.security.token.Token;
159 
160 import com.google.common.collect.ArrayListMultimap;
161 import com.google.common.collect.ListMultimap;
162 import com.google.common.collect.Lists;
163 import com.google.protobuf.ByteString;
164 import com.google.protobuf.CodedInputStream;
165 import com.google.protobuf.InvalidProtocolBufferException;
166 import com.google.protobuf.Message;
167 import com.google.protobuf.Parser;
168 import com.google.protobuf.RpcChannel;
169 import com.google.protobuf.Service;
170 import com.google.protobuf.ServiceException;
171 import com.google.protobuf.TextFormat;
172 
173 /**
174  * Protobufs utility.
175  */
176 @edu.umd.cs.findbugs.annotations.SuppressWarnings(
177   value="DP_CREATE_CLASSLOADER_INSIDE_DO_PRIVILEGED", justification="None. Address sometime.")
178 @InterfaceAudience.Private // TODO: some clients (Hive, etc) use this class
179 public final class ProtobufUtil {
180 
181   private ProtobufUtil() {
182   }
183 
184   /**
185    * Primitive type to class mapping.
186    */
187   private final static Map<String, Class<?>>
188     PRIMITIVES = new HashMap<String, Class<?>>();
189 
190   /**
191    * Many results are simple: no cell, exists true or false. To save on object creations,
192    *  we reuse them across calls.
193    */
194   private final static Cell[] EMPTY_CELL_ARRAY = new Cell[]{};
195   private final static Result EMPTY_RESULT = Result.create(EMPTY_CELL_ARRAY);
196   private final static Result EMPTY_RESULT_EXISTS_TRUE = Result.create(null, true);
197   private final static Result EMPTY_RESULT_EXISTS_FALSE = Result.create(null, false);
198   private final static Result EMPTY_RESULT_STALE = Result.create(EMPTY_CELL_ARRAY, null, true);
199   private final static Result EMPTY_RESULT_EXISTS_TRUE_STALE
200     = Result.create((Cell[])null, true, true);
201   private final static Result EMPTY_RESULT_EXISTS_FALSE_STALE
202     = Result.create((Cell[])null, false, true);
203 
204   private final static ClientProtos.Result EMPTY_RESULT_PB;
205   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_TRUE;
206   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_FALSE;
207   private final static ClientProtos.Result EMPTY_RESULT_PB_STALE;
208   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_TRUE_STALE;
209   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_FALSE_STALE;
210 
211 
212   static {
213     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
214 
215     builder.setExists(true);
216     builder.setAssociatedCellCount(0);
217     EMPTY_RESULT_PB_EXISTS_TRUE =  builder.build();
218 
219     builder.setStale(true);
220     EMPTY_RESULT_PB_EXISTS_TRUE_STALE = builder.build();
221     builder.clear();
222 
223     builder.setExists(false);
224     builder.setAssociatedCellCount(0);
225     EMPTY_RESULT_PB_EXISTS_FALSE =  builder.build();
226     builder.setStale(true);
227     EMPTY_RESULT_PB_EXISTS_FALSE_STALE = builder.build();
228 
229     builder.clear();
230     builder.setAssociatedCellCount(0);
231     EMPTY_RESULT_PB =  builder.build();
232     builder.setStale(true);
233     EMPTY_RESULT_PB_STALE = builder.build();
234   }
235 
236   /**
237    * Dynamic class loader to load filter/comparators
238    */
239   private final static ClassLoader CLASS_LOADER;
240 
241   static {
242     ClassLoader parent = ProtobufUtil.class.getClassLoader();
243     Configuration conf = HBaseConfiguration.create();
244     CLASS_LOADER = new DynamicClassLoader(conf, parent);
245 
246     PRIMITIVES.put(Boolean.TYPE.getName(), Boolean.TYPE);
247     PRIMITIVES.put(Byte.TYPE.getName(), Byte.TYPE);
248     PRIMITIVES.put(Character.TYPE.getName(), Character.TYPE);
249     PRIMITIVES.put(Short.TYPE.getName(), Short.TYPE);
250     PRIMITIVES.put(Integer.TYPE.getName(), Integer.TYPE);
251     PRIMITIVES.put(Long.TYPE.getName(), Long.TYPE);
252     PRIMITIVES.put(Float.TYPE.getName(), Float.TYPE);
253     PRIMITIVES.put(Double.TYPE.getName(), Double.TYPE);
254     PRIMITIVES.put(Void.TYPE.getName(), Void.TYPE);
255   }
256 
257   /**
258    * Prepend the passed bytes with four bytes of magic, {@link ProtobufMagic#PB_MAGIC},
259    * to flag what follows as a protobuf in hbase.  Prepend these bytes to all content written to
260    * znodes, etc.
261    * @param bytes Bytes to decorate
262    * @return The passed <code>bytes</code> with magic prepended (Creates a new
263    * byte array that is <code>bytes.length</code> plus {@link ProtobufMagic#PB_MAGIC}.length.
264    */
265   public static byte [] prependPBMagic(final byte [] bytes) {
266     return Bytes.add(ProtobufMagic.PB_MAGIC, bytes);
267   }
268 
269   /**
270    * @param bytes Bytes to check.
271    * @return True if passed <code>bytes</code> has {@link ProtobufMagic#PB_MAGIC} for a prefix.
272    */
273   public static boolean isPBMagicPrefix(final byte [] bytes) {
274     return ProtobufMagic.isPBMagicPrefix(bytes);
275   }
276 
277   /**
278    * @param bytes Bytes to check.
279    * @param offset offset to start at
280    * @param len length to use
281    * @return True if passed <code>bytes</code> has {@link ProtobufMagic#PB_MAGIC} for a prefix.
282    */
283   public static boolean isPBMagicPrefix(final byte [] bytes, int offset, int len) {
284     return ProtobufMagic.isPBMagicPrefix(bytes, offset, len);
285   }
286 
287   /**
288    * @param bytes bytes to check
289    * @throws DeserializationException if we are missing the pb magic prefix
290    */
291   public static void expectPBMagicPrefix(final byte [] bytes) throws DeserializationException {
292     if (!isPBMagicPrefix(bytes)) {
293       throw new DeserializationException("Missing pb magic " +
294           Bytes.toString(ProtobufMagic.PB_MAGIC) + " prefix");
295     }
296   }
297 
298   /**
299    * @return Length of {@link ProtobufMagic#lengthOfPBMagic()}
300    */
301   public static int lengthOfPBMagic() {
302     return ProtobufMagic.lengthOfPBMagic();
303   }
304 
305   /**
306    * Return the IOException thrown by the remote server wrapped in
307    * ServiceException as cause.
308    *
309    * @param se ServiceException that wraps IO exception thrown by the server
310    * @return Exception wrapped in ServiceException or
311    *   a new IOException that wraps the unexpected ServiceException.
312    */
313   public static IOException getRemoteException(ServiceException se) {
314     Throwable e = se.getCause();
315     if (e == null) {
316       return new IOException(se);
317     }
318     if (ExceptionUtil.isInterrupt(e)) {
319       return ExceptionUtil.asInterrupt(e);
320     }
321     if (e instanceof RemoteException) {
322       e = ((RemoteException) e).unwrapRemoteException();
323     }
324     return e instanceof IOException ? (IOException) e : new IOException(se);
325   }
326 
327   /**
328    * Convert a ServerName to a protocol buffer ServerName
329    *
330    * @param serverName the ServerName to convert
331    * @return the converted protocol buffer ServerName
332    * @see #toServerName(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName)
333    */
334   public static HBaseProtos.ServerName
335       toServerName(final ServerName serverName) {
336     if (serverName == null) return null;
337     HBaseProtos.ServerName.Builder builder =
338       HBaseProtos.ServerName.newBuilder();
339     builder.setHostName(serverName.getHostname());
340     if (serverName.getPort() >= 0) {
341       builder.setPort(serverName.getPort());
342     }
343     if (serverName.getStartcode() >= 0) {
344       builder.setStartCode(serverName.getStartcode());
345     }
346     return builder.build();
347   }
348 
349   /**
350    * Convert a protocol buffer ServerName to a ServerName
351    *
352    * @param proto the protocol buffer ServerName to convert
353    * @return the converted ServerName
354    */
355   public static ServerName toServerName(final HBaseProtos.ServerName proto) {
356     if (proto == null) return null;
357     String hostName = proto.getHostName();
358     long startCode = -1;
359     int port = -1;
360     if (proto.hasPort()) {
361       port = proto.getPort();
362     }
363     if (proto.hasStartCode()) {
364       startCode = proto.getStartCode();
365     }
366     return ServerName.valueOf(hostName, port, startCode);
367   }
368 
369   /**
370    * Get HTableDescriptor[] from GetTableDescriptorsResponse protobuf
371    *
372    * @param proto the GetTableDescriptorsResponse
373    * @return HTableDescriptor[]
374    */
375   public static HTableDescriptor[] getHTableDescriptorArray(GetTableDescriptorsResponse proto) {
376     if (proto == null) return null;
377 
378     HTableDescriptor[] ret = new HTableDescriptor[proto.getTableSchemaCount()];
379     for (int i = 0; i < proto.getTableSchemaCount(); ++i) {
380       ret[i] = HTableDescriptor.convert(proto.getTableSchema(i));
381     }
382     return ret;
383   }
384 
385   /**
386    * get the split keys in form "byte [][]" from a CreateTableRequest proto
387    *
388    * @param proto the CreateTableRequest
389    * @return the split keys
390    */
391   public static byte [][] getSplitKeysArray(final CreateTableRequest proto) {
392     byte [][] splitKeys = new byte[proto.getSplitKeysCount()][];
393     for (int i = 0; i < proto.getSplitKeysCount(); ++i) {
394       splitKeys[i] = proto.getSplitKeys(i).toByteArray();
395     }
396     return splitKeys;
397   }
398 
399   /**
400    * Convert a protobuf Durability into a client Durability
401    */
402   public static Durability toDurability(
403       final ClientProtos.MutationProto.Durability proto) {
404     switch(proto) {
405     case USE_DEFAULT:
406       return Durability.USE_DEFAULT;
407     case SKIP_WAL:
408       return Durability.SKIP_WAL;
409     case ASYNC_WAL:
410       return Durability.ASYNC_WAL;
411     case SYNC_WAL:
412       return Durability.SYNC_WAL;
413     case FSYNC_WAL:
414       return Durability.FSYNC_WAL;
415     default:
416       return Durability.USE_DEFAULT;
417     }
418   }
419 
420   /**
421    * Convert a client Durability into a protbuf Durability
422    */
423   public static ClientProtos.MutationProto.Durability toDurability(
424       final Durability d) {
425     switch(d) {
426     case USE_DEFAULT:
427       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
428     case SKIP_WAL:
429       return ClientProtos.MutationProto.Durability.SKIP_WAL;
430     case ASYNC_WAL:
431       return ClientProtos.MutationProto.Durability.ASYNC_WAL;
432     case SYNC_WAL:
433       return ClientProtos.MutationProto.Durability.SYNC_WAL;
434     case FSYNC_WAL:
435       return ClientProtos.MutationProto.Durability.FSYNC_WAL;
436     default:
437       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
438     }
439   }
440 
441   /**
442    * Convert a protocol buffer Get to a client Get
443    *
444    * @param proto the protocol buffer Get to convert
445    * @return the converted client Get
446    * @throws IOException
447    */
448   public static Get toGet(
449       final ClientProtos.Get proto) throws IOException {
450     if (proto == null) return null;
451     byte[] row = proto.getRow().toByteArray();
452     Get get = new Get(row);
453     if (proto.hasCacheBlocks()) {
454       get.setCacheBlocks(proto.getCacheBlocks());
455     }
456     if (proto.hasMaxVersions()) {
457       get.setMaxVersions(proto.getMaxVersions());
458     }
459     if (proto.hasStoreLimit()) {
460       get.setMaxResultsPerColumnFamily(proto.getStoreLimit());
461     }
462     if (proto.hasStoreOffset()) {
463       get.setRowOffsetPerColumnFamily(proto.getStoreOffset());
464     }
465     if (proto.getCfTimeRangeCount() > 0) {
466       for (HBaseProtos.ColumnFamilyTimeRange cftr : proto.getCfTimeRangeList()) {
467         TimeRange timeRange = protoToTimeRange(cftr.getTimeRange());
468         get.setColumnFamilyTimeRange(cftr.getColumnFamily().toByteArray(),
469             timeRange.getMin(), timeRange.getMax());
470       }
471     }
472     if (proto.hasTimeRange()) {
473       TimeRange timeRange = protoToTimeRange(proto.getTimeRange());
474       get.setTimeRange(timeRange.getMin(), timeRange.getMax());
475     }
476     if (proto.hasFilter()) {
477       FilterProtos.Filter filter = proto.getFilter();
478       get.setFilter(ProtobufUtil.toFilter(filter));
479     }
480     for (NameBytesPair attribute: proto.getAttributeList()) {
481       get.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
482     }
483     if (proto.getColumnCount() > 0) {
484       for (Column column: proto.getColumnList()) {
485         byte[] family = column.getFamily().toByteArray();
486         if (column.getQualifierCount() > 0) {
487           for (ByteString qualifier: column.getQualifierList()) {
488             get.addColumn(family, qualifier.toByteArray());
489           }
490         } else {
491           get.addFamily(family);
492         }
493       }
494     }
495     if (proto.hasExistenceOnly() && proto.getExistenceOnly()){
496       get.setCheckExistenceOnly(true);
497     }
498     if (proto.hasConsistency()) {
499       get.setConsistency(toConsistency(proto.getConsistency()));
500     }
501     return get;
502   }
503 
504   public static Consistency toConsistency(ClientProtos.Consistency consistency) {
505     switch (consistency) {
506       case STRONG : return Consistency.STRONG;
507       case TIMELINE : return Consistency.TIMELINE;
508       default : return Consistency.STRONG;
509     }
510   }
511 
512   public static ClientProtos.Consistency toConsistency(Consistency consistency) {
513     switch (consistency) {
514       case STRONG : return ClientProtos.Consistency.STRONG;
515       case TIMELINE : return ClientProtos.Consistency.TIMELINE;
516       default : return ClientProtos.Consistency.STRONG;
517     }
518   }
519 
520   /**
521    * Convert a protocol buffer Mutate to a Put.
522    *
523    * @param proto The protocol buffer MutationProto to convert
524    * @return A client Put.
525    * @throws IOException
526    */
527   public static Put toPut(final MutationProto proto)
528   throws IOException {
529     return toPut(proto, null);
530   }
531 
532   /**
533    * Convert a protocol buffer Mutate to a Put.
534    *
535    * @param proto The protocol buffer MutationProto to convert
536    * @param cellScanner If non-null, the Cell data that goes with this proto.
537    * @return A client Put.
538    * @throws IOException
539    */
540   public static Put toPut(final MutationProto proto, final CellScanner cellScanner)
541   throws IOException {
542     // TODO: Server-side at least why do we convert back to the Client types?  Why not just pb it?
543     MutationType type = proto.getMutateType();
544     assert type == MutationType.PUT: type.name();
545     long timestamp = proto.hasTimestamp()? proto.getTimestamp(): HConstants.LATEST_TIMESTAMP;
546     Put put = proto.hasRow() ? new Put(proto.getRow().toByteArray(), timestamp) : null;
547     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
548     if (cellCount > 0) {
549       // The proto has metadata only and the data is separate to be found in the cellScanner.
550       if (cellScanner == null) {
551         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
552             toShortString(proto));
553       }
554       for (int i = 0; i < cellCount; i++) {
555         if (!cellScanner.advance()) {
556           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
557             " no cell returned: " + toShortString(proto));
558         }
559         Cell cell = cellScanner.current();
560         if (put == null) {
561           put = new Put(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
562         }
563         put.add(cell);
564       }
565     } else {
566       if (put == null) {
567         throw new IllegalArgumentException("row cannot be null");
568       }
569       // The proto has the metadata and the data itself
570       for (ColumnValue column: proto.getColumnValueList()) {
571         byte[] family = column.getFamily().toByteArray();
572         for (QualifierValue qv: column.getQualifierValueList()) {
573           if (!qv.hasValue()) {
574             throw new DoNotRetryIOException(
575                 "Missing required field: qualifier value");
576           }
577           ByteBuffer qualifier =
578               qv.hasQualifier() ? qv.getQualifier().asReadOnlyByteBuffer() : null;
579           ByteBuffer value =
580               qv.hasValue() ? qv.getValue().asReadOnlyByteBuffer() : null;
581           long ts = timestamp;
582           if (qv.hasTimestamp()) {
583             ts = qv.getTimestamp();
584           }
585           byte[] allTagsBytes;
586           if (qv.hasTags()) {
587             allTagsBytes = qv.getTags().toByteArray();
588             if(qv.hasDeleteType()) {
589               byte[] qual = qv.hasQualifier() ? qv.getQualifier().toByteArray() : null;
590               put.add(new KeyValue(proto.getRow().toByteArray(), family, qual, ts,
591                   fromDeleteType(qv.getDeleteType()), null, allTagsBytes));
592             } else {
593               List<Tag> tags = TagUtil.asList(allTagsBytes, 0, (short)allTagsBytes.length);
594               Tag[] tagsArray = new Tag[tags.size()];
595               put.addImmutable(family, qualifier, ts, value, tags.toArray(tagsArray));
596             }
597           } else {
598             if(qv.hasDeleteType()) {
599               byte[] qual = qv.hasQualifier() ? qv.getQualifier().toByteArray() : null;
600               put.add(new KeyValue(proto.getRow().toByteArray(), family, qual, ts,
601                   fromDeleteType(qv.getDeleteType())));
602             } else{
603               put.addImmutable(family, qualifier, ts, value);
604             }
605           }
606         }
607       }
608     }
609     put.setDurability(toDurability(proto.getDurability()));
610     for (NameBytesPair attribute: proto.getAttributeList()) {
611       put.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
612     }
613     return put;
614   }
615 
616   /**
617    * Convert a protocol buffer Mutate to a Delete
618    *
619    * @param proto the protocol buffer Mutate to convert
620    * @return the converted client Delete
621    * @throws IOException
622    */
623   public static Delete toDelete(final MutationProto proto)
624   throws IOException {
625     return toDelete(proto, null);
626   }
627 
628   /**
629    * Convert a protocol buffer Mutate to a Delete
630    *
631    * @param proto the protocol buffer Mutate to convert
632    * @param cellScanner if non-null, the data that goes with this delete.
633    * @return the converted client Delete
634    * @throws IOException
635    */
636   public static Delete toDelete(final MutationProto proto, final CellScanner cellScanner)
637   throws IOException {
638     MutationType type = proto.getMutateType();
639     assert type == MutationType.DELETE : type.name();
640     long timestamp = proto.hasTimestamp() ? proto.getTimestamp() : HConstants.LATEST_TIMESTAMP;
641     Delete delete = proto.hasRow() ? new Delete(proto.getRow().toByteArray(), timestamp) : null;
642     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
643     if (cellCount > 0) {
644       // The proto has metadata only and the data is separate to be found in the cellScanner.
645       if (cellScanner == null) {
646         // TextFormat should be fine for a Delete since it carries no data, just coordinates.
647         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
648           TextFormat.shortDebugString(proto));
649       }
650       for (int i = 0; i < cellCount; i++) {
651         if (!cellScanner.advance()) {
652           // TextFormat should be fine for a Delete since it carries no data, just coordinates.
653           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
654             " no cell returned: " + TextFormat.shortDebugString(proto));
655         }
656         Cell cell = cellScanner.current();
657         if (delete == null) {
658           delete =
659             new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
660         }
661         delete.addDeleteMarker(cell);
662       }
663     } else {
664       if (delete == null) {
665         throw new IllegalArgumentException("row cannot be null");
666       }
667       for (ColumnValue column: proto.getColumnValueList()) {
668         byte[] family = column.getFamily().toByteArray();
669         for (QualifierValue qv: column.getQualifierValueList()) {
670           DeleteType deleteType = qv.getDeleteType();
671           byte[] qualifier = null;
672           if (qv.hasQualifier()) {
673             qualifier = qv.getQualifier().toByteArray();
674           }
675           long ts = HConstants.LATEST_TIMESTAMP;
676           if (qv.hasTimestamp()) {
677             ts = qv.getTimestamp();
678           }
679           if (deleteType == DeleteType.DELETE_ONE_VERSION) {
680             delete.addColumn(family, qualifier, ts);
681           } else if (deleteType == DeleteType.DELETE_MULTIPLE_VERSIONS) {
682             delete.addColumns(family, qualifier, ts);
683           } else if (deleteType == DeleteType.DELETE_FAMILY_VERSION) {
684             delete.addFamilyVersion(family, ts);
685           } else {
686             delete.addFamily(family, ts);
687           }
688         }
689       }
690     }
691     delete.setDurability(toDurability(proto.getDurability()));
692     for (NameBytesPair attribute: proto.getAttributeList()) {
693       delete.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
694     }
695     return delete;
696   }
697 
698   /**
699    * Convert a protocol buffer Mutate to an Append
700    * @param cellScanner
701    * @param proto the protocol buffer Mutate to convert
702    * @return the converted client Append
703    * @throws IOException
704    */
705   public static Append toAppend(final MutationProto proto, final CellScanner cellScanner)
706   throws IOException {
707     MutationType type = proto.getMutateType();
708     assert type == MutationType.APPEND : type.name();
709     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
710     Append append = null;
711     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
712     if (cellCount > 0) {
713       // The proto has metadata only and the data is separate to be found in the cellScanner.
714       if (cellScanner == null) {
715         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
716           toShortString(proto));
717       }
718       for (int i = 0; i < cellCount; i++) {
719         if (!cellScanner.advance()) {
720           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
721             " no cell returned: " + toShortString(proto));
722         }
723         Cell cell = cellScanner.current();
724         if (append == null) {
725           append = new Append(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
726         }
727         append.add(cell);
728       }
729     } else {
730       append = new Append(row);
731       for (ColumnValue column: proto.getColumnValueList()) {
732         byte[] family = column.getFamily().toByteArray();
733         for (QualifierValue qv: column.getQualifierValueList()) {
734           byte[] qualifier = qv.getQualifier().toByteArray();
735           if (!qv.hasValue()) {
736             throw new DoNotRetryIOException(
737               "Missing required field: qualifier value");
738           }
739           byte[] value = qv.getValue().toByteArray();
740           byte[] tags = null;
741           if (qv.hasTags()) {
742             tags = qv.getTags().toByteArray();
743           }
744           append.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
745               KeyValue.Type.Put, value, tags));
746         }
747       }
748     }
749     append.setDurability(toDurability(proto.getDurability()));
750     for (NameBytesPair attribute: proto.getAttributeList()) {
751       append.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
752     }
753     return append;
754   }
755 
756   /**
757    * Convert a MutateRequest to Mutation
758    *
759    * @param proto the protocol buffer Mutate to convert
760    * @return the converted Mutation
761    * @throws IOException
762    */
763   public static Mutation toMutation(final MutationProto proto) throws IOException {
764     MutationType type = proto.getMutateType();
765     if (type == MutationType.APPEND) {
766       return toAppend(proto, null);
767     }
768     if (type == MutationType.DELETE) {
769       return toDelete(proto, null);
770     }
771     if (type == MutationType.PUT) {
772       return toPut(proto, null);
773     }
774     throw new IOException("Unknown mutation type " + type);
775   }
776 
777   /**
778    * Convert a protocol buffer Mutate to an Increment
779    *
780    * @param proto the protocol buffer Mutate to convert
781    * @return the converted client Increment
782    * @throws IOException
783    */
784   public static Increment toIncrement(final MutationProto proto, final CellScanner cellScanner)
785   throws IOException {
786     MutationType type = proto.getMutateType();
787     assert type == MutationType.INCREMENT : type.name();
788     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
789     Increment increment = null;
790     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
791     if (cellCount > 0) {
792       // The proto has metadata only and the data is separate to be found in the cellScanner.
793       if (cellScanner == null) {
794         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
795           TextFormat.shortDebugString(proto));
796       }
797       for (int i = 0; i < cellCount; i++) {
798         if (!cellScanner.advance()) {
799           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
800             " no cell returned: " + TextFormat.shortDebugString(proto));
801         }
802         Cell cell = cellScanner.current();
803         if (increment == null) {
804           increment = new Increment(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
805         }
806         increment.add(cell);
807       }
808     } else {
809       increment = new Increment(row);
810       for (ColumnValue column: proto.getColumnValueList()) {
811         byte[] family = column.getFamily().toByteArray();
812         for (QualifierValue qv: column.getQualifierValueList()) {
813           byte[] qualifier = qv.getQualifier().toByteArray();
814           if (!qv.hasValue()) {
815             throw new DoNotRetryIOException("Missing required field: qualifier value");
816           }
817           byte[] value = qv.getValue().toByteArray();
818           byte[] tags = null;
819           if (qv.hasTags()) {
820             tags = qv.getTags().toByteArray();
821           }
822           increment.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
823               KeyValue.Type.Put, value, tags));
824         }
825       }
826     }
827     if (proto.hasTimeRange()) {
828       TimeRange timeRange = protoToTimeRange(proto.getTimeRange());
829       increment.setTimeRange(timeRange.getMin(), timeRange.getMax());
830     }
831     increment.setDurability(toDurability(proto.getDurability()));
832     for (NameBytesPair attribute : proto.getAttributeList()) {
833       increment.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
834     }
835     return increment;
836   }
837 
838   /**
839    * Convert a client Scan to a protocol buffer Scan
840    *
841    * @param scan the client Scan to convert
842    * @return the converted protocol buffer Scan
843    * @throws IOException
844    */
845   public static ClientProtos.Scan toScan(
846       final Scan scan) throws IOException {
847     ClientProtos.Scan.Builder scanBuilder =
848       ClientProtos.Scan.newBuilder();
849     scanBuilder.setCacheBlocks(scan.getCacheBlocks());
850     if (scan.getBatch() > 0) {
851       scanBuilder.setBatchSize(scan.getBatch());
852     }
853     if (scan.getMaxResultSize() > 0) {
854       scanBuilder.setMaxResultSize(scan.getMaxResultSize());
855     }
856     if (scan.isSmall()) {
857       scanBuilder.setSmall(scan.isSmall());
858     }
859     if (scan.getAllowPartialResults()) {
860       scanBuilder.setAllowPartialResults(scan.getAllowPartialResults());
861     }
862     Boolean loadColumnFamiliesOnDemand = scan.getLoadColumnFamiliesOnDemandValue();
863     if (loadColumnFamiliesOnDemand != null) {
864       scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand.booleanValue());
865     }
866     scanBuilder.setMaxVersions(scan.getMaxVersions());
867     for (Entry<byte[], TimeRange> cftr : scan.getColumnFamilyTimeRange().entrySet()) {
868       HBaseProtos.ColumnFamilyTimeRange.Builder b = HBaseProtos.ColumnFamilyTimeRange.newBuilder();
869       b.setColumnFamily(ByteString.copyFrom(cftr.getKey()));
870       b.setTimeRange(timeRangeToProto(cftr.getValue()));
871       scanBuilder.addCfTimeRange(b);
872     }
873     TimeRange timeRange = scan.getTimeRange();
874     if (!timeRange.isAllTime()) {
875       HBaseProtos.TimeRange.Builder timeRangeBuilder =
876         HBaseProtos.TimeRange.newBuilder();
877       timeRangeBuilder.setFrom(timeRange.getMin());
878       timeRangeBuilder.setTo(timeRange.getMax());
879       scanBuilder.setTimeRange(timeRangeBuilder.build());
880     }
881     Map<String, byte[]> attributes = scan.getAttributesMap();
882     if (!attributes.isEmpty()) {
883       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
884       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
885         attributeBuilder.setName(attribute.getKey());
886         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
887         scanBuilder.addAttribute(attributeBuilder.build());
888       }
889     }
890     byte[] startRow = scan.getStartRow();
891     if (startRow != null && startRow.length > 0) {
892       scanBuilder.setStartRow(ByteStringer.wrap(startRow));
893     }
894     byte[] stopRow = scan.getStopRow();
895     if (stopRow != null && stopRow.length > 0) {
896       scanBuilder.setStopRow(ByteStringer.wrap(stopRow));
897     }
898     if (scan.hasFilter()) {
899       scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
900     }
901     if (scan.hasFamilies()) {
902       Column.Builder columnBuilder = Column.newBuilder();
903       for (Map.Entry<byte[],NavigableSet<byte []>>
904           family: scan.getFamilyMap().entrySet()) {
905         columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
906         NavigableSet<byte []> qualifiers = family.getValue();
907         columnBuilder.clearQualifier();
908         if (qualifiers != null && qualifiers.size() > 0) {
909           for (byte [] qualifier: qualifiers) {
910             columnBuilder.addQualifier(ByteStringer.wrap(qualifier));
911           }
912         }
913         scanBuilder.addColumn(columnBuilder.build());
914       }
915     }
916     if (scan.getMaxResultsPerColumnFamily() >= 0) {
917       scanBuilder.setStoreLimit(scan.getMaxResultsPerColumnFamily());
918     }
919     if (scan.getRowOffsetPerColumnFamily() > 0) {
920       scanBuilder.setStoreOffset(scan.getRowOffsetPerColumnFamily());
921     }
922     if (scan.isReversed()) {
923       scanBuilder.setReversed(scan.isReversed());
924     }
925     if (scan.getConsistency() == Consistency.TIMELINE) {
926       scanBuilder.setConsistency(toConsistency(scan.getConsistency()));
927     }
928     if (scan.getCaching() > 0) {
929       scanBuilder.setCaching(scan.getCaching());
930     }
931     return scanBuilder.build();
932   }
933 
934   /**
935    * Convert a protocol buffer Scan to a client Scan
936    *
937    * @param proto the protocol buffer Scan to convert
938    * @return the converted client Scan
939    * @throws IOException
940    */
941   public static Scan toScan(
942       final ClientProtos.Scan proto) throws IOException {
943     byte [] startRow = HConstants.EMPTY_START_ROW;
944     byte [] stopRow  = HConstants.EMPTY_END_ROW;
945     if (proto.hasStartRow()) {
946       startRow = proto.getStartRow().toByteArray();
947     }
948     if (proto.hasStopRow()) {
949       stopRow = proto.getStopRow().toByteArray();
950     }
951     Scan scan = new Scan(startRow, stopRow);
952     if (proto.hasCacheBlocks()) {
953       scan.setCacheBlocks(proto.getCacheBlocks());
954     }
955     if (proto.hasMaxVersions()) {
956       scan.setMaxVersions(proto.getMaxVersions());
957     }
958     if (proto.hasStoreLimit()) {
959       scan.setMaxResultsPerColumnFamily(proto.getStoreLimit());
960     }
961     if (proto.hasStoreOffset()) {
962       scan.setRowOffsetPerColumnFamily(proto.getStoreOffset());
963     }
964     if (proto.hasLoadColumnFamiliesOnDemand()) {
965       scan.setLoadColumnFamiliesOnDemand(proto.getLoadColumnFamiliesOnDemand());
966     }
967     if (proto.getCfTimeRangeCount() > 0) {
968       for (HBaseProtos.ColumnFamilyTimeRange cftr : proto.getCfTimeRangeList()) {
969         TimeRange timeRange = protoToTimeRange(cftr.getTimeRange());
970         scan.setColumnFamilyTimeRange(cftr.getColumnFamily().toByteArray(),
971             timeRange.getMin(), timeRange.getMax());
972       }
973     }
974     if (proto.hasTimeRange()) {
975       TimeRange timeRange = protoToTimeRange(proto.getTimeRange());
976       scan.setTimeRange(timeRange.getMin(), timeRange.getMax());
977     }
978     if (proto.hasFilter()) {
979       FilterProtos.Filter filter = proto.getFilter();
980       scan.setFilter(ProtobufUtil.toFilter(filter));
981     }
982     if (proto.hasBatchSize()) {
983       scan.setBatch(proto.getBatchSize());
984     }
985     if (proto.hasMaxResultSize()) {
986       scan.setMaxResultSize(proto.getMaxResultSize());
987     }
988     if (proto.hasSmall()) {
989       scan.setSmall(proto.getSmall());
990     }
991     if (proto.hasAllowPartialResults()) {
992       scan.setAllowPartialResults(proto.getAllowPartialResults());
993     }
994     for (NameBytesPair attribute: proto.getAttributeList()) {
995       scan.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
996     }
997     if (proto.getColumnCount() > 0) {
998       for (Column column: proto.getColumnList()) {
999         byte[] family = column.getFamily().toByteArray();
1000         if (column.getQualifierCount() > 0) {
1001           for (ByteString qualifier: column.getQualifierList()) {
1002             scan.addColumn(family, qualifier.toByteArray());
1003           }
1004         } else {
1005           scan.addFamily(family);
1006         }
1007       }
1008     }
1009     if (proto.hasReversed()) {
1010       scan.setReversed(proto.getReversed());
1011     }
1012     if (proto.hasConsistency()) {
1013       scan.setConsistency(toConsistency(proto.getConsistency()));
1014     }
1015     if (proto.hasCaching()) {
1016       scan.setCaching(proto.getCaching());
1017     }
1018     return scan;
1019   }
1020 
1021   /**
1022    * Create a protocol buffer Get based on a client Get.
1023    *
1024    * @param get the client Get
1025    * @return a protocol buffer Get
1026    * @throws IOException
1027    */
1028   public static ClientProtos.Get toGet(
1029       final Get get) throws IOException {
1030     ClientProtos.Get.Builder builder =
1031       ClientProtos.Get.newBuilder();
1032     builder.setRow(ByteStringer.wrap(get.getRow()));
1033     builder.setCacheBlocks(get.getCacheBlocks());
1034     builder.setMaxVersions(get.getMaxVersions());
1035     if (get.getFilter() != null) {
1036       builder.setFilter(ProtobufUtil.toFilter(get.getFilter()));
1037     }
1038     for (Entry<byte[], TimeRange> cftr : get.getColumnFamilyTimeRange().entrySet()) {
1039       HBaseProtos.ColumnFamilyTimeRange.Builder b = HBaseProtos.ColumnFamilyTimeRange.newBuilder();
1040       b.setColumnFamily(ByteString.copyFrom(cftr.getKey()));
1041       b.setTimeRange(timeRangeToProto(cftr.getValue()));
1042       builder.addCfTimeRange(b);
1043     }
1044     TimeRange timeRange = get.getTimeRange();
1045     if (!timeRange.isAllTime()) {
1046       HBaseProtos.TimeRange.Builder timeRangeBuilder =
1047         HBaseProtos.TimeRange.newBuilder();
1048       timeRangeBuilder.setFrom(timeRange.getMin());
1049       timeRangeBuilder.setTo(timeRange.getMax());
1050       builder.setTimeRange(timeRangeBuilder.build());
1051     }
1052     Map<String, byte[]> attributes = get.getAttributesMap();
1053     if (!attributes.isEmpty()) {
1054       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1055       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
1056         attributeBuilder.setName(attribute.getKey());
1057         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1058         builder.addAttribute(attributeBuilder.build());
1059       }
1060     }
1061     if (get.hasFamilies()) {
1062       Column.Builder columnBuilder = Column.newBuilder();
1063       Map<byte[], NavigableSet<byte[]>> families = get.getFamilyMap();
1064       for (Map.Entry<byte[], NavigableSet<byte[]>> family: families.entrySet()) {
1065         NavigableSet<byte[]> qualifiers = family.getValue();
1066         columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1067         columnBuilder.clearQualifier();
1068         if (qualifiers != null && qualifiers.size() > 0) {
1069           for (byte[] qualifier: qualifiers) {
1070             columnBuilder.addQualifier(ByteStringer.wrap(qualifier));
1071           }
1072         }
1073         builder.addColumn(columnBuilder.build());
1074       }
1075     }
1076     if (get.getMaxResultsPerColumnFamily() >= 0) {
1077       builder.setStoreLimit(get.getMaxResultsPerColumnFamily());
1078     }
1079     if (get.getRowOffsetPerColumnFamily() > 0) {
1080       builder.setStoreOffset(get.getRowOffsetPerColumnFamily());
1081     }
1082     if (get.isCheckExistenceOnly()){
1083       builder.setExistenceOnly(true);
1084     }
1085     if (get.getConsistency() != null && get.getConsistency() != Consistency.STRONG) {
1086       builder.setConsistency(toConsistency(get.getConsistency()));
1087     }
1088 
1089     return builder.build();
1090   }
1091 
1092   /**
1093    * Convert a client Increment to a protobuf Mutate.
1094    *
1095    * @param increment
1096    * @return the converted mutate
1097    */
1098   public static MutationProto toMutation(
1099     final Increment increment, final MutationProto.Builder builder, long nonce) {
1100     builder.setRow(ByteStringer.wrap(increment.getRow()));
1101     builder.setMutateType(MutationType.INCREMENT);
1102     builder.setDurability(toDurability(increment.getDurability()));
1103     if (nonce != HConstants.NO_NONCE) {
1104       builder.setNonce(nonce);
1105     }
1106     TimeRange timeRange = increment.getTimeRange();
1107     if (!timeRange.isAllTime()) {
1108       HBaseProtos.TimeRange.Builder timeRangeBuilder =
1109         HBaseProtos.TimeRange.newBuilder();
1110       timeRangeBuilder.setFrom(timeRange.getMin());
1111       timeRangeBuilder.setTo(timeRange.getMax());
1112       builder.setTimeRange(timeRangeBuilder.build());
1113     }
1114     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1115     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1116     for (Map.Entry<byte[], List<Cell>> family: increment.getFamilyCellMap().entrySet()) {
1117       columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1118       columnBuilder.clearQualifierValue();
1119       List<Cell> values = family.getValue();
1120       if (values != null && values.size() > 0) {
1121         for (Cell cell: values) {
1122           valueBuilder.clear();
1123           valueBuilder.setQualifier(ByteStringer.wrap(
1124               cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
1125           valueBuilder.setValue(ByteStringer.wrap(
1126               cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
1127           if (cell.getTagsLength() > 0) {
1128             valueBuilder.setTags(ByteStringer.wrap(cell.getTagsArray(),
1129                 cell.getTagsOffset(), cell.getTagsLength()));
1130           }
1131           columnBuilder.addQualifierValue(valueBuilder.build());
1132         }
1133       }
1134       builder.addColumnValue(columnBuilder.build());
1135     }
1136     Map<String, byte[]> attributes = increment.getAttributesMap();
1137     if (!attributes.isEmpty()) {
1138       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1139       for (Map.Entry<String, byte[]> attribute : attributes.entrySet()) {
1140         attributeBuilder.setName(attribute.getKey());
1141         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1142         builder.addAttribute(attributeBuilder.build());
1143       }
1144     }
1145     return builder.build();
1146   }
1147 
1148   public static MutationProto toMutation(final MutationType type, final Mutation mutation)
1149     throws IOException {
1150     return toMutation(type, mutation, HConstants.NO_NONCE);
1151   }
1152 
1153   /**
1154    * Create a protocol buffer Mutate based on a client Mutation
1155    *
1156    * @param type
1157    * @param mutation
1158    * @return a protobuf'd Mutation
1159    * @throws IOException
1160    */
1161   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1162     final long nonce) throws IOException {
1163     return toMutation(type, mutation, MutationProto.newBuilder(), nonce);
1164   }
1165 
1166   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1167       MutationProto.Builder builder) throws IOException {
1168     return toMutation(type, mutation, builder, HConstants.NO_NONCE);
1169   }
1170 
1171   @SuppressWarnings("deprecation")
1172   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1173       MutationProto.Builder builder, long nonce)
1174   throws IOException {
1175     builder = getMutationBuilderAndSetCommonFields(type, mutation, builder);
1176     if (nonce != HConstants.NO_NONCE) {
1177       builder.setNonce(nonce);
1178     }
1179     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1180     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1181     for (Map.Entry<byte[],List<Cell>> family: mutation.getFamilyCellMap().entrySet()) {
1182       columnBuilder.clear();
1183       columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1184       for (Cell cell: family.getValue()) {
1185         valueBuilder.clear();
1186         valueBuilder.setQualifier(ByteStringer.wrap(
1187             cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
1188         valueBuilder.setValue(ByteStringer.wrap(
1189             cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
1190         valueBuilder.setTimestamp(cell.getTimestamp());
1191         if(cell.getTagsLength() > 0) {
1192           valueBuilder.setTags(ByteStringer.wrap(cell.getTagsArray(), cell.getTagsOffset(),
1193               cell.getTagsLength()));
1194         }
1195         if (type == MutationType.DELETE || (type == MutationType.PUT && CellUtil.isDelete(cell))) {
1196           KeyValue.Type keyValueType = KeyValue.Type.codeToType(cell.getTypeByte());
1197           valueBuilder.setDeleteType(toDeleteType(keyValueType));
1198         }
1199         columnBuilder.addQualifierValue(valueBuilder.build());
1200       }
1201       builder.addColumnValue(columnBuilder.build());
1202     }
1203     return builder.build();
1204   }
1205 
1206   /**
1207    * Create a protocol buffer MutationProto based on a client Mutation. Does NOT include data.
1208    * Understanding is that the Cell will be transported other than via protobuf.
1209    * @param type
1210    * @param mutation
1211    * @param builder
1212    * @return a protobuf'd Mutation
1213    * @throws IOException
1214    */
1215   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1216       final MutationProto.Builder builder)  throws IOException {
1217     return toMutationNoData(type, mutation, builder, HConstants.NO_NONCE);
1218   }
1219 
1220   /**
1221    * Create a protocol buffer MutationProto based on a client Mutation.  Does NOT include data.
1222    * Understanding is that the Cell will be transported other than via protobuf.
1223    * @param type
1224    * @param mutation
1225    * @return a protobuf'd Mutation
1226    * @throws IOException
1227    */
1228   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation)
1229   throws IOException {
1230     MutationProto.Builder builder =  MutationProto.newBuilder();
1231     return toMutationNoData(type, mutation, builder);
1232   }
1233 
1234   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1235       final MutationProto.Builder builder, long nonce) throws IOException {
1236     getMutationBuilderAndSetCommonFields(type, mutation, builder);
1237     builder.setAssociatedCellCount(mutation.size());
1238     if (nonce != HConstants.NO_NONCE) {
1239       builder.setNonce(nonce);
1240     }
1241     return builder.build();
1242   }
1243 
1244   /**
1245    * Code shared by {@link #toMutation(MutationType, Mutation)} and
1246    * {@link #toMutationNoData(MutationType, Mutation)}
1247    * @param type
1248    * @param mutation
1249    * @return A partly-filled out protobuf'd Mutation.
1250    */
1251   private static MutationProto.Builder getMutationBuilderAndSetCommonFields(final MutationType type,
1252       final Mutation mutation, MutationProto.Builder builder) {
1253     builder.setRow(ByteStringer.wrap(mutation.getRow()));
1254     builder.setMutateType(type);
1255     builder.setDurability(toDurability(mutation.getDurability()));
1256     builder.setTimestamp(mutation.getTimeStamp());
1257     Map<String, byte[]> attributes = mutation.getAttributesMap();
1258     if (!attributes.isEmpty()) {
1259       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1260       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
1261         attributeBuilder.setName(attribute.getKey());
1262         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1263         builder.addAttribute(attributeBuilder.build());
1264       }
1265     }
1266     return builder;
1267   }
1268 
1269   /**
1270    * Convert a client Result to a protocol buffer Result
1271    *
1272    * @param result the client Result to convert
1273    * @return the converted protocol buffer Result
1274    */
1275   public static ClientProtos.Result toResult(final Result result) {
1276     if (result.getExists() != null) {
1277       return toResult(result.getExists(), result.isStale());
1278     }
1279 
1280     Cell[] cells = result.rawCells();
1281     if (cells == null || cells.length == 0) {
1282       return result.isStale() ? EMPTY_RESULT_PB_STALE : EMPTY_RESULT_PB;
1283     }
1284 
1285     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1286     for (Cell c : cells) {
1287       builder.addCell(toCell(c));
1288     }
1289 
1290     builder.setStale(result.isStale());
1291     builder.setPartial(result.isPartial());
1292 
1293     return builder.build();
1294   }
1295 
1296   /**
1297    * Convert a client Result to a protocol buffer Result
1298    *
1299    * @param existence the client existence to send
1300    * @return the converted protocol buffer Result
1301    */
1302   public static ClientProtos.Result toResult(final boolean existence, boolean stale) {
1303     if (stale){
1304       return existence ? EMPTY_RESULT_PB_EXISTS_TRUE_STALE : EMPTY_RESULT_PB_EXISTS_FALSE_STALE;
1305     } else {
1306       return existence ? EMPTY_RESULT_PB_EXISTS_TRUE : EMPTY_RESULT_PB_EXISTS_FALSE;
1307     }
1308   }
1309 
1310   /**
1311    * Convert a client Result to a protocol buffer Result.
1312    * The pb Result does not include the Cell data.  That is for transport otherwise.
1313    *
1314    * @param result the client Result to convert
1315    * @return the converted protocol buffer Result
1316    */
1317   public static ClientProtos.Result toResultNoData(final Result result) {
1318     if (result.getExists() != null) return toResult(result.getExists(), result.isStale());
1319     int size = result.size();
1320     if (size == 0) return result.isStale() ? EMPTY_RESULT_PB_STALE : EMPTY_RESULT_PB;
1321     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1322     builder.setAssociatedCellCount(size);
1323     builder.setStale(result.isStale());
1324     return builder.build();
1325   }
1326 
1327   /**
1328    * Convert a protocol buffer Result to a client Result
1329    *
1330    * @param proto the protocol buffer Result to convert
1331    * @return the converted client Result
1332    */
1333   public static Result toResult(final ClientProtos.Result proto) {
1334     if (proto.hasExists()) {
1335       if (proto.getStale()) {
1336         return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE_STALE :EMPTY_RESULT_EXISTS_FALSE_STALE;
1337       }
1338       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1339     }
1340 
1341     List<CellProtos.Cell> values = proto.getCellList();
1342     if (values.isEmpty()){
1343       return proto.getStale() ? EMPTY_RESULT_STALE : EMPTY_RESULT;
1344     }
1345 
1346     List<Cell> cells = new ArrayList<Cell>(values.size());
1347     for (CellProtos.Cell c : values) {
1348       cells.add(toCell(c));
1349     }
1350     return Result.create(cells, null, proto.getStale(), proto.getPartial());
1351   }
1352 
1353   /**
1354    * Convert a protocol buffer Result to a client Result
1355    *
1356    * @param proto the protocol buffer Result to convert
1357    * @param scanner Optional cell scanner.
1358    * @return the converted client Result
1359    * @throws IOException
1360    */
1361   public static Result toResult(final ClientProtos.Result proto, final CellScanner scanner)
1362   throws IOException {
1363     List<CellProtos.Cell> values = proto.getCellList();
1364 
1365     if (proto.hasExists()) {
1366       if ((values != null && !values.isEmpty()) ||
1367           (proto.hasAssociatedCellCount() && proto.getAssociatedCellCount() > 0)) {
1368         throw new IllegalArgumentException("bad proto: exists with cells is no allowed " + proto);
1369       }
1370       if (proto.getStale()) {
1371         return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE_STALE :EMPTY_RESULT_EXISTS_FALSE_STALE;
1372       }
1373       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1374     }
1375 
1376     // TODO: Unit test that has some Cells in scanner and some in the proto.
1377     List<Cell> cells = null;
1378     if (proto.hasAssociatedCellCount()) {
1379       int count = proto.getAssociatedCellCount();
1380       cells = new ArrayList<Cell>(count + values.size());
1381       for (int i = 0; i < count; i++) {
1382         if (!scanner.advance()) throw new IOException("Failed get " + i + " of " + count);
1383         cells.add(scanner.current());
1384       }
1385     }
1386 
1387     if (!values.isEmpty()){
1388       if (cells == null) cells = new ArrayList<Cell>(values.size());
1389       for (CellProtos.Cell c: values) {
1390         cells.add(toCell(c));
1391       }
1392     }
1393 
1394     return (cells == null || cells.isEmpty())
1395         ? (proto.getStale() ? EMPTY_RESULT_STALE : EMPTY_RESULT)
1396         : Result.create(cells, null, proto.getStale());
1397   }
1398 
1399 
1400   /**
1401    * Convert a ByteArrayComparable to a protocol buffer Comparator
1402    *
1403    * @param comparator the ByteArrayComparable to convert
1404    * @return the converted protocol buffer Comparator
1405    */
1406   public static ComparatorProtos.Comparator toComparator(ByteArrayComparable comparator) {
1407     ComparatorProtos.Comparator.Builder builder = ComparatorProtos.Comparator.newBuilder();
1408     builder.setName(comparator.getClass().getName());
1409     builder.setSerializedComparator(ByteStringer.wrap(comparator.toByteArray()));
1410     return builder.build();
1411   }
1412 
1413   /**
1414    * Convert a protocol buffer Comparator to a ByteArrayComparable
1415    *
1416    * @param proto the protocol buffer Comparator to convert
1417    * @return the converted ByteArrayComparable
1418    */
1419   @SuppressWarnings("unchecked")
1420   public static ByteArrayComparable toComparator(ComparatorProtos.Comparator proto)
1421   throws IOException {
1422     String type = proto.getName();
1423     String funcName = "parseFrom";
1424     byte [] value = proto.getSerializedComparator().toByteArray();
1425     try {
1426       Class<? extends ByteArrayComparable> c =
1427         (Class<? extends ByteArrayComparable>)Class.forName(type, true, CLASS_LOADER);
1428       Method parseFrom = c.getMethod(funcName, byte[].class);
1429       if (parseFrom == null) {
1430         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1431       }
1432       return (ByteArrayComparable)parseFrom.invoke(null, value);
1433     } catch (Exception e) {
1434       throw new IOException(e);
1435     }
1436   }
1437 
1438   /**
1439    * Convert a protocol buffer Filter to a client Filter
1440    *
1441    * @param proto the protocol buffer Filter to convert
1442    * @return the converted Filter
1443    */
1444   @SuppressWarnings("unchecked")
1445   public static Filter toFilter(FilterProtos.Filter proto) throws IOException {
1446     String type = proto.getName();
1447     final byte [] value = proto.getSerializedFilter().toByteArray();
1448     String funcName = "parseFrom";
1449     try {
1450       Class<? extends Filter> c =
1451         (Class<? extends Filter>)Class.forName(type, true, CLASS_LOADER);
1452       Method parseFrom = c.getMethod(funcName, byte[].class);
1453       if (parseFrom == null) {
1454         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1455       }
1456       return (Filter)parseFrom.invoke(c, value);
1457     } catch (Exception e) {
1458       // Either we couldn't instantiate the method object, or "parseFrom" failed.
1459       // In either case, let's not retry.
1460       throw new DoNotRetryIOException(e);
1461     }
1462   }
1463 
1464   /**
1465    * Convert a client Filter to a protocol buffer Filter
1466    *
1467    * @param filter the Filter to convert
1468    * @return the converted protocol buffer Filter
1469    */
1470   public static FilterProtos.Filter toFilter(Filter filter) throws IOException {
1471     FilterProtos.Filter.Builder builder = FilterProtos.Filter.newBuilder();
1472     builder.setName(filter.getClass().getName());
1473     builder.setSerializedFilter(ByteStringer.wrap(filter.toByteArray()));
1474     return builder.build();
1475   }
1476 
1477   /**
1478    * Convert a delete KeyValue type to protocol buffer DeleteType.
1479    *
1480    * @param type
1481    * @return protocol buffer DeleteType
1482    * @throws IOException
1483    */
1484   public static DeleteType toDeleteType(
1485       KeyValue.Type type) throws IOException {
1486     switch (type) {
1487     case Delete:
1488       return DeleteType.DELETE_ONE_VERSION;
1489     case DeleteColumn:
1490       return DeleteType.DELETE_MULTIPLE_VERSIONS;
1491     case DeleteFamily:
1492       return DeleteType.DELETE_FAMILY;
1493     case DeleteFamilyVersion:
1494       return DeleteType.DELETE_FAMILY_VERSION;
1495     default:
1496         throw new IOException("Unknown delete type: " + type);
1497     }
1498   }
1499 
1500   /**
1501    * Convert a protocol buffer DeleteType to delete KeyValue type.
1502    *
1503    * @param type The DeleteType
1504    * @return The type.
1505    * @throws IOException
1506    */
1507   public static KeyValue.Type fromDeleteType(
1508       DeleteType type) throws IOException {
1509     switch (type) {
1510     case DELETE_ONE_VERSION:
1511       return KeyValue.Type.Delete;
1512     case DELETE_MULTIPLE_VERSIONS:
1513       return KeyValue.Type.DeleteColumn;
1514     case DELETE_FAMILY:
1515       return KeyValue.Type.DeleteFamily;
1516     case DELETE_FAMILY_VERSION:
1517       return KeyValue.Type.DeleteFamilyVersion;
1518     default:
1519       throw new IOException("Unknown delete type: " + type);
1520     }
1521   }
1522 
1523   /**
1524    * Convert a stringified protocol buffer exception Parameter to a Java Exception
1525    *
1526    * @param parameter the protocol buffer Parameter to convert
1527    * @return the converted Exception
1528    * @throws IOException if failed to deserialize the parameter
1529    */
1530   @SuppressWarnings("unchecked")
1531   public static Throwable toException(final NameBytesPair parameter) throws IOException {
1532     if (parameter == null || !parameter.hasValue()) return null;
1533     String desc = parameter.getValue().toStringUtf8();
1534     String type = parameter.getName();
1535     try {
1536       Class<? extends Throwable> c =
1537         (Class<? extends Throwable>)Class.forName(type, true, CLASS_LOADER);
1538       Constructor<? extends Throwable> cn = null;
1539       try {
1540         cn = c.getDeclaredConstructor(String.class);
1541         return cn.newInstance(desc);
1542       } catch (NoSuchMethodException e) {
1543         // Could be a raw RemoteException. See HBASE-8987.
1544         cn = c.getDeclaredConstructor(String.class, String.class);
1545         return cn.newInstance(type, desc);
1546       }
1547     } catch (Exception e) {
1548       throw new IOException(e);
1549     }
1550   }
1551 
1552 // Start helpers for Client
1553 
1554   /**
1555    * A helper to bulk load a list of HFiles using client protocol.
1556    *
1557    * @param client
1558    * @param familyPaths
1559    * @param regionName
1560    * @param assignSeqNum
1561    * @return true if all are loaded
1562    * @throws IOException
1563    */
1564   public static boolean bulkLoadHFile(final ClientService.BlockingInterface client,
1565       final List<Pair<byte[], String>> familyPaths,
1566       final byte[] regionName, boolean assignSeqNum) throws IOException {
1567     BulkLoadHFileRequest request =
1568       RequestConverter.buildBulkLoadHFileRequest(familyPaths, regionName, assignSeqNum);
1569     try {
1570       BulkLoadHFileResponse response =
1571         client.bulkLoadHFile(null, request);
1572       return response.getLoaded();
1573     } catch (ServiceException se) {
1574       throw getRemoteException(se);
1575     }
1576   }
1577 
1578   public static CoprocessorServiceResponse execService(final ClientService.BlockingInterface client,
1579       final CoprocessorServiceCall call, final byte[] regionName) throws IOException {
1580     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1581         .setCall(call).setRegion(
1582             RequestConverter.buildRegionSpecifier(REGION_NAME, regionName)).build();
1583     try {
1584       CoprocessorServiceResponse response =
1585           client.execService(null, request);
1586       return response;
1587     } catch (ServiceException se) {
1588       throw getRemoteException(se);
1589     }
1590   }
1591 
1592   public static CoprocessorServiceResponse execService(
1593     final MasterService.BlockingInterface client, final CoprocessorServiceCall call)
1594   throws IOException {
1595     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1596         .setCall(call).setRegion(
1597            RequestConverter.buildRegionSpecifier(REGION_NAME, HConstants.EMPTY_BYTE_ARRAY)).build();
1598     try {
1599       CoprocessorServiceResponse response =
1600           client.execMasterService(null, request);
1601       return response;
1602     } catch (ServiceException se) {
1603       throw getRemoteException(se);
1604     }
1605   }
1606 
1607   /**
1608    * Make a region server endpoint call
1609    * @param client
1610    * @param call
1611    * @return CoprocessorServiceResponse
1612    * @throws IOException
1613    */
1614   public static CoprocessorServiceResponse execRegionServerService(
1615       final ClientService.BlockingInterface client, final CoprocessorServiceCall call)
1616       throws IOException {
1617     CoprocessorServiceRequest request =
1618         CoprocessorServiceRequest
1619             .newBuilder()
1620             .setCall(call)
1621             .setRegion(
1622               RequestConverter.buildRegionSpecifier(REGION_NAME, HConstants.EMPTY_BYTE_ARRAY))
1623             .build();
1624     try {
1625       CoprocessorServiceResponse response = client.execRegionServerService(null, request);
1626       return response;
1627     } catch (ServiceException se) {
1628       throw getRemoteException(se);
1629     }
1630   }
1631 
1632   @SuppressWarnings("unchecked")
1633   public static <T extends Service> T newServiceStub(Class<T> service, RpcChannel channel)
1634       throws Exception {
1635     return (T)Methods.call(service, null, "newStub",
1636         new Class[]{ RpcChannel.class }, new Object[]{ channel });
1637   }
1638 
1639 // End helpers for Client
1640 // Start helpers for Admin
1641 
1642   /**
1643    * A helper to retrieve region info given a region name
1644    * using admin protocol.
1645    *
1646    * @param admin
1647    * @param regionName
1648    * @return the retrieved region info
1649    * @throws IOException
1650    */
1651   public static HRegionInfo getRegionInfo(final AdminService.BlockingInterface admin,
1652       final byte[] regionName) throws IOException {
1653     try {
1654       GetRegionInfoRequest request =
1655         RequestConverter.buildGetRegionInfoRequest(regionName);
1656       GetRegionInfoResponse response =
1657         admin.getRegionInfo(null, request);
1658       return HRegionInfo.convert(response.getRegionInfo());
1659     } catch (ServiceException se) {
1660       throw getRemoteException(se);
1661     }
1662   }
1663 
1664   /**
1665    * A helper to close a region given a region name
1666    * using admin protocol.
1667    *
1668    * @param admin
1669    * @param regionName
1670    * @throws IOException
1671    */
1672   public static void closeRegion(final AdminService.BlockingInterface admin,
1673       final ServerName server, final byte[] regionName) throws IOException {
1674     CloseRegionRequest closeRegionRequest =
1675       RequestConverter.buildCloseRegionRequest(server, regionName);
1676     try {
1677       admin.closeRegion(null, closeRegionRequest);
1678     } catch (ServiceException se) {
1679       throw getRemoteException(se);
1680     }
1681   }
1682 
1683   /**
1684    * A helper to close a region given a region name
1685    * using admin protocol.
1686    *
1687    * @param admin
1688    * @param regionName
1689    * @return true if the region is closed
1690    * @throws IOException
1691    */
1692   public static boolean closeRegion(final AdminService.BlockingInterface admin,
1693       final ServerName server, final byte[] regionName,
1694       final ServerName destinationServer) throws IOException {
1695     CloseRegionRequest closeRegionRequest =
1696       RequestConverter.buildCloseRegionRequest(server,
1697         regionName, destinationServer);
1698     try {
1699       CloseRegionResponse response = admin.closeRegion(null, closeRegionRequest);
1700       return ResponseConverter.isClosed(response);
1701     } catch (ServiceException se) {
1702       throw getRemoteException(se);
1703     }
1704   }
1705 
1706   /**
1707    * A helper to warmup a region given a region name
1708    * using admin protocol
1709    *
1710    * @param admin
1711    * @param regionInfo
1712    *
1713    */
1714   public static void warmupRegion(final AdminService.BlockingInterface admin,
1715       final HRegionInfo regionInfo) throws IOException  {
1716 
1717     try {
1718       WarmupRegionRequest warmupRegionRequest =
1719            RequestConverter.buildWarmupRegionRequest(regionInfo);
1720 
1721       admin.warmupRegion(null, warmupRegionRequest);
1722     } catch (ServiceException e) {
1723       throw getRemoteException(e);
1724     }
1725   }
1726 
1727   /**
1728    * A helper to open a region using admin protocol.
1729    * @param admin
1730    * @param region
1731    * @throws IOException
1732    */
1733   public static void openRegion(final AdminService.BlockingInterface admin,
1734       ServerName server, final HRegionInfo region) throws IOException {
1735     OpenRegionRequest request =
1736       RequestConverter.buildOpenRegionRequest(server, region, null, null);
1737     try {
1738       admin.openRegion(null, request);
1739     } catch (ServiceException se) {
1740       throw ProtobufUtil.getRemoteException(se);
1741     }
1742   }
1743 
1744 
1745   /**
1746    * A helper to get the all the online regions on a region
1747    * server using admin protocol.
1748    *
1749    * @param admin
1750    * @return a list of online region info
1751    * @throws IOException
1752    */
1753   public static List<HRegionInfo> getOnlineRegions(final AdminService.BlockingInterface admin)
1754   throws IOException {
1755     GetOnlineRegionRequest request = RequestConverter.buildGetOnlineRegionRequest();
1756     GetOnlineRegionResponse response = null;
1757     try {
1758       response = admin.getOnlineRegion(null, request);
1759     } catch (ServiceException se) {
1760       throw getRemoteException(se);
1761     }
1762     return getRegionInfos(response);
1763   }
1764 
1765   /**
1766    * Get the list of region info from a GetOnlineRegionResponse
1767    *
1768    * @param proto the GetOnlineRegionResponse
1769    * @return the list of region info or null if <code>proto</code> is null
1770    */
1771   static List<HRegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
1772     if (proto == null) return null;
1773     List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
1774     for (RegionInfo regionInfo: proto.getRegionInfoList()) {
1775       regionInfos.add(HRegionInfo.convert(regionInfo));
1776     }
1777     return regionInfos;
1778   }
1779 
1780   /**
1781    * A helper to get the info of a region server using admin protocol.
1782    *
1783    * @param admin
1784    * @return the server name
1785    * @throws IOException
1786    */
1787   public static ServerInfo getServerInfo(final AdminService.BlockingInterface admin)
1788   throws IOException {
1789     GetServerInfoRequest request = RequestConverter.buildGetServerInfoRequest();
1790     try {
1791       GetServerInfoResponse response = admin.getServerInfo(null, request);
1792       return response.getServerInfo();
1793     } catch (ServiceException se) {
1794       throw getRemoteException(se);
1795     }
1796   }
1797 
1798   /**
1799    * A helper to get the list of files of a column family
1800    * on a given region using admin protocol.
1801    *
1802    * @param admin
1803    * @param regionName
1804    * @param family
1805    * @return the list of store files
1806    * @throws IOException
1807    */
1808   public static List<String> getStoreFiles(final AdminService.BlockingInterface admin,
1809       final byte[] regionName, final byte[] family)
1810   throws IOException {
1811     GetStoreFileRequest request =
1812       RequestConverter.buildGetStoreFileRequest(regionName, family);
1813     try {
1814       GetStoreFileResponse response = admin.getStoreFile(null, request);
1815       return response.getStoreFileList();
1816     } catch (ServiceException se) {
1817       throw ProtobufUtil.getRemoteException(se);
1818     }
1819   }
1820 
1821   /**
1822    * A helper to split a region using admin protocol.
1823    *
1824    * @param admin
1825    * @param hri
1826    * @param splitPoint
1827    * @throws IOException
1828    */
1829   public static void split(final AdminService.BlockingInterface admin,
1830       final HRegionInfo hri, byte[] splitPoint) throws IOException {
1831     SplitRegionRequest request =
1832       RequestConverter.buildSplitRegionRequest(hri.getRegionName(), splitPoint);
1833     try {
1834       admin.splitRegion(null, request);
1835     } catch (ServiceException se) {
1836       throw ProtobufUtil.getRemoteException(se);
1837     }
1838   }
1839 
1840   /**
1841    * A helper to merge regions using admin protocol. Send request to
1842    * regionserver.
1843    * @param admin
1844    * @param region_a
1845    * @param region_b
1846    * @param forcible true if do a compulsory merge, otherwise we will only merge
1847    *          two adjacent regions
1848    * @param user effective user
1849    * @throws IOException
1850    */
1851   public static void mergeRegions(final AdminService.BlockingInterface admin,
1852       final HRegionInfo region_a, final HRegionInfo region_b,
1853       final boolean forcible, final User user) throws IOException {
1854     final MergeRegionsRequest request = RequestConverter.buildMergeRegionsRequest(
1855         region_a.getRegionName(), region_b.getRegionName(),forcible);
1856     if (user != null) {
1857       try {
1858         user.getUGI().doAs(new PrivilegedExceptionAction<Void>() {
1859           @Override
1860           public Void run() throws Exception {
1861             admin.mergeRegions(null, request);
1862             return null;
1863           }
1864         });
1865       } catch (InterruptedException ie) {
1866         InterruptedIOException iioe = new InterruptedIOException();
1867         iioe.initCause(ie);
1868         throw iioe;
1869       }
1870     } else {
1871       try {
1872         admin.mergeRegions(null, request);
1873       } catch (ServiceException se) {
1874         throw ProtobufUtil.getRemoteException(se);
1875       }
1876     }
1877   }
1878 
1879 // End helpers for Admin
1880 
1881   /*
1882    * Get the total (read + write) requests from a RegionLoad pb
1883    * @param rl - RegionLoad pb
1884    * @return total (read + write) requests
1885    */
1886   public static long getTotalRequestsCount(RegionLoad rl) {
1887     if (rl == null) {
1888       return 0;
1889     }
1890 
1891     return rl.getReadRequestsCount() + rl.getWriteRequestsCount();
1892   }
1893 
1894 
1895   /**
1896    * @param m Message to get delimited pb serialization of (with pb magic prefix)
1897    */
1898   public static byte [] toDelimitedByteArray(final Message m) throws IOException {
1899     // Allocate arbitrary big size so we avoid resizing.
1900     ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
1901     baos.write(ProtobufMagic.PB_MAGIC);
1902     m.writeDelimitedTo(baos);
1903     return baos.toByteArray();
1904   }
1905 
1906   /**
1907    * Converts a Permission proto to a client Permission object.
1908    *
1909    * @param proto the protobuf Permission
1910    * @return the converted Permission
1911    */
1912   public static Permission toPermission(AccessControlProtos.Permission proto) {
1913     if (proto.getType() != AccessControlProtos.Permission.Type.Global) {
1914       return toTablePermission(proto);
1915     } else {
1916       List<Permission.Action> actions = toPermissionActions(
1917         proto.getGlobalPermission().getActionList());
1918       return new Permission(actions.toArray(new Permission.Action[actions.size()]));
1919     }
1920   }
1921 
1922   /**
1923    * Converts a Permission proto to a client TablePermission object.
1924    *
1925    * @param proto the protobuf Permission
1926    * @return the converted TablePermission
1927    */
1928   public static TablePermission toTablePermission(AccessControlProtos.Permission proto) {
1929     if(proto.getType() == AccessControlProtos.Permission.Type.Global) {
1930       AccessControlProtos.GlobalPermission perm = proto.getGlobalPermission();
1931       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1932 
1933       return new TablePermission(null, null, null,
1934           actions.toArray(new Permission.Action[actions.size()]));
1935     }
1936     if(proto.getType() == AccessControlProtos.Permission.Type.Namespace) {
1937       AccessControlProtos.NamespacePermission perm = proto.getNamespacePermission();
1938       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1939 
1940       if(!proto.hasNamespacePermission()) {
1941         throw new IllegalStateException("Namespace must not be empty in NamespacePermission");
1942       }
1943       String namespace = perm.getNamespaceName().toStringUtf8();
1944       return new TablePermission(namespace, actions.toArray(new Permission.Action[actions.size()]));
1945     }
1946     if(proto.getType() == AccessControlProtos.Permission.Type.Table) {
1947       AccessControlProtos.TablePermission perm = proto.getTablePermission();
1948       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1949 
1950       byte[] qualifier = null;
1951       byte[] family = null;
1952       TableName table = null;
1953 
1954       if (!perm.hasTableName()) {
1955         throw new IllegalStateException("TableName cannot be empty");
1956       }
1957       table = ProtobufUtil.toTableName(perm.getTableName());
1958 
1959       if (perm.hasFamily()) family = perm.getFamily().toByteArray();
1960       if (perm.hasQualifier()) qualifier = perm.getQualifier().toByteArray();
1961 
1962       return new TablePermission(table, family, qualifier,
1963           actions.toArray(new Permission.Action[actions.size()]));
1964     }
1965     throw new IllegalStateException("Unrecognize Perm Type: "+proto.getType());
1966   }
1967 
1968   /**
1969    * Convert a client Permission to a Permission proto
1970    *
1971    * @param perm the client Permission
1972    * @return the protobuf Permission
1973    */
1974   public static AccessControlProtos.Permission toPermission(Permission perm) {
1975     AccessControlProtos.Permission.Builder ret = AccessControlProtos.Permission.newBuilder();
1976     if (perm instanceof TablePermission) {
1977       TablePermission tablePerm = (TablePermission)perm;
1978       if(tablePerm.hasNamespace()) {
1979         ret.setType(AccessControlProtos.Permission.Type.Namespace);
1980 
1981         AccessControlProtos.NamespacePermission.Builder builder =
1982             AccessControlProtos.NamespacePermission.newBuilder();
1983         builder.setNamespaceName(ByteString.copyFromUtf8(tablePerm.getNamespace()));
1984         Permission.Action[] actions = perm.getActions();
1985         if (actions != null) {
1986           for (Permission.Action a : actions) {
1987             builder.addAction(toPermissionAction(a));
1988           }
1989         }
1990         ret.setNamespacePermission(builder);
1991         return ret.build();
1992       } else if (tablePerm.hasTable()) {
1993         ret.setType(AccessControlProtos.Permission.Type.Table);
1994 
1995         AccessControlProtos.TablePermission.Builder builder =
1996             AccessControlProtos.TablePermission.newBuilder();
1997         builder.setTableName(ProtobufUtil.toProtoTableName(tablePerm.getTableName()));
1998         if (tablePerm.hasFamily()) {
1999           builder.setFamily(ByteStringer.wrap(tablePerm.getFamily()));
2000         }
2001         if (tablePerm.hasQualifier()) {
2002           builder.setQualifier(ByteStringer.wrap(tablePerm.getQualifier()));
2003         }
2004         Permission.Action actions[] = perm.getActions();
2005         if (actions != null) {
2006           for (Permission.Action a : actions) {
2007             builder.addAction(toPermissionAction(a));
2008           }
2009         }
2010         ret.setTablePermission(builder);
2011         return ret.build();
2012       }
2013     }
2014 
2015     ret.setType(AccessControlProtos.Permission.Type.Global);
2016 
2017     AccessControlProtos.GlobalPermission.Builder builder =
2018         AccessControlProtos.GlobalPermission.newBuilder();
2019     Permission.Action actions[] = perm.getActions();
2020     if (actions != null) {
2021       for (Permission.Action a: actions) {
2022         builder.addAction(toPermissionAction(a));
2023       }
2024     }
2025     ret.setGlobalPermission(builder);
2026     return ret.build();
2027   }
2028 
2029   /**
2030    * Converts a list of Permission.Action proto to a list of client Permission.Action objects.
2031    *
2032    * @param protoActions the list of protobuf Actions
2033    * @return the converted list of Actions
2034    */
2035   public static List<Permission.Action> toPermissionActions(
2036       List<AccessControlProtos.Permission.Action> protoActions) {
2037     List<Permission.Action> actions = new ArrayList<Permission.Action>(protoActions.size());
2038     for (AccessControlProtos.Permission.Action a : protoActions) {
2039       actions.add(toPermissionAction(a));
2040     }
2041     return actions;
2042   }
2043 
2044   /**
2045    * Converts a Permission.Action proto to a client Permission.Action object.
2046    *
2047    * @param action the protobuf Action
2048    * @return the converted Action
2049    */
2050   public static Permission.Action toPermissionAction(
2051       AccessControlProtos.Permission.Action action) {
2052     switch (action) {
2053       case READ:
2054         return Permission.Action.READ;
2055       case WRITE:
2056         return Permission.Action.WRITE;
2057       case EXEC:
2058         return Permission.Action.EXEC;
2059       case CREATE:
2060         return Permission.Action.CREATE;
2061       case ADMIN:
2062         return Permission.Action.ADMIN;
2063     }
2064     throw new IllegalArgumentException("Unknown action value "+action.name());
2065   }
2066 
2067   /**
2068    * Convert a client Permission.Action to a Permission.Action proto
2069    *
2070    * @param action the client Action
2071    * @return the protobuf Action
2072    */
2073   public static AccessControlProtos.Permission.Action toPermissionAction(
2074       Permission.Action action) {
2075     switch (action) {
2076       case READ:
2077         return AccessControlProtos.Permission.Action.READ;
2078       case WRITE:
2079         return AccessControlProtos.Permission.Action.WRITE;
2080       case EXEC:
2081         return AccessControlProtos.Permission.Action.EXEC;
2082       case CREATE:
2083         return AccessControlProtos.Permission.Action.CREATE;
2084       case ADMIN:
2085         return AccessControlProtos.Permission.Action.ADMIN;
2086     }
2087     throw new IllegalArgumentException("Unknown action value "+action.name());
2088   }
2089 
2090   /**
2091    * Convert a client user permission to a user permission proto
2092    *
2093    * @param perm the client UserPermission
2094    * @return the protobuf UserPermission
2095    */
2096   public static AccessControlProtos.UserPermission toUserPermission(UserPermission perm) {
2097     return AccessControlProtos.UserPermission.newBuilder()
2098         .setUser(ByteStringer.wrap(perm.getUser()))
2099         .setPermission(toPermission(perm))
2100         .build();
2101   }
2102 
2103   /**
2104    * Converts a user permission proto to a client user permission object.
2105    *
2106    * @param proto the protobuf UserPermission
2107    * @return the converted UserPermission
2108    */
2109   public static UserPermission toUserPermission(AccessControlProtos.UserPermission proto) {
2110     return new UserPermission(proto.getUser().toByteArray(),
2111         toTablePermission(proto.getPermission()));
2112   }
2113 
2114   /**
2115    * Convert a ListMultimap&lt;String, TablePermission&gt; where key is username
2116    * to a protobuf UserPermission
2117    *
2118    * @param perm the list of user and table permissions
2119    * @return the protobuf UserTablePermissions
2120    */
2121   public static AccessControlProtos.UsersAndPermissions toUserTablePermissions(
2122       ListMultimap<String, TablePermission> perm) {
2123     AccessControlProtos.UsersAndPermissions.Builder builder =
2124                   AccessControlProtos.UsersAndPermissions.newBuilder();
2125     for (Map.Entry<String, Collection<TablePermission>> entry : perm.asMap().entrySet()) {
2126       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
2127                   AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
2128       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
2129       for (TablePermission tablePerm: entry.getValue()) {
2130         userPermBuilder.addPermissions(toPermission(tablePerm));
2131       }
2132       builder.addUserPermissions(userPermBuilder.build());
2133     }
2134     return builder.build();
2135   }
2136 
2137   /**
2138    * A utility used to grant a user global permissions.
2139    * <p>
2140    * It's also called by the shell, in case you want to find references.
2141    *
2142    * @param protocol the AccessControlService protocol proxy
2143    * @param userShortName the short name of the user to grant permissions
2144    * @param actions the permissions to be granted
2145    * @throws ServiceException
2146    */
2147   public static void grant(AccessControlService.BlockingInterface protocol,
2148       String userShortName, Permission.Action... actions) throws ServiceException {
2149     List<AccessControlProtos.Permission.Action> permActions =
2150         Lists.newArrayListWithCapacity(actions.length);
2151     for (Permission.Action a : actions) {
2152       permActions.add(ProtobufUtil.toPermissionAction(a));
2153     }
2154     AccessControlProtos.GrantRequest request = RequestConverter.
2155       buildGrantRequest(userShortName, permActions.toArray(
2156         new AccessControlProtos.Permission.Action[actions.length]));
2157     protocol.grant(null, request);
2158   }
2159 
2160   /**
2161    * A utility used to grant a user table permissions. The permissions will
2162    * be for a table table/column family/qualifier.
2163    * <p>
2164    * It's also called by the shell, in case you want to find references.
2165    *
2166    * @param protocol the AccessControlService protocol proxy
2167    * @param userShortName the short name of the user to grant permissions
2168    * @param tableName optional table name
2169    * @param f optional column family
2170    * @param q optional qualifier
2171    * @param actions the permissions to be granted
2172    * @throws ServiceException
2173    */
2174   public static void grant(AccessControlService.BlockingInterface protocol,
2175       String userShortName, TableName tableName, byte[] f, byte[] q,
2176       Permission.Action... actions) throws ServiceException {
2177     List<AccessControlProtos.Permission.Action> permActions =
2178         Lists.newArrayListWithCapacity(actions.length);
2179     for (Permission.Action a : actions) {
2180       permActions.add(ProtobufUtil.toPermissionAction(a));
2181     }
2182     AccessControlProtos.GrantRequest request = RequestConverter.
2183       buildGrantRequest(userShortName, tableName, f, q, permActions.toArray(
2184         new AccessControlProtos.Permission.Action[actions.length]));
2185     protocol.grant(null, request);
2186   }
2187 
2188   /**
2189    * A utility used to grant a user namespace permissions.
2190    * <p>
2191    * It's also called by the shell, in case you want to find references.
2192    *
2193    * @param protocol the AccessControlService protocol proxy
2194    * @param namespace the short name of the user to grant permissions
2195    * @param actions the permissions to be granted
2196    * @throws ServiceException
2197    */
2198   public static void grant(AccessControlService.BlockingInterface protocol,
2199       String userShortName, String namespace,
2200       Permission.Action... actions) throws ServiceException {
2201     List<AccessControlProtos.Permission.Action> permActions =
2202         Lists.newArrayListWithCapacity(actions.length);
2203     for (Permission.Action a : actions) {
2204       permActions.add(ProtobufUtil.toPermissionAction(a));
2205     }
2206     AccessControlProtos.GrantRequest request = RequestConverter.
2207       buildGrantRequest(userShortName, namespace, permActions.toArray(
2208         new AccessControlProtos.Permission.Action[actions.length]));
2209     protocol.grant(null, request);
2210   }
2211 
2212   /**
2213    * A utility used to revoke a user's global permissions.
2214    * <p>
2215    * It's also called by the shell, in case you want to find references.
2216    *
2217    * @param protocol the AccessControlService protocol proxy
2218    * @param userShortName the short name of the user to revoke permissions
2219    * @param actions the permissions to be revoked
2220    * @throws ServiceException
2221    */
2222   public static void revoke(AccessControlService.BlockingInterface protocol,
2223       String userShortName, Permission.Action... actions) throws ServiceException {
2224     List<AccessControlProtos.Permission.Action> permActions =
2225         Lists.newArrayListWithCapacity(actions.length);
2226     for (Permission.Action a : actions) {
2227       permActions.add(ProtobufUtil.toPermissionAction(a));
2228     }
2229     AccessControlProtos.RevokeRequest request = RequestConverter.
2230       buildRevokeRequest(userShortName, permActions.toArray(
2231         new AccessControlProtos.Permission.Action[actions.length]));
2232     protocol.revoke(null, request);
2233   }
2234 
2235   /**
2236    * A utility used to revoke a user's table permissions. The permissions will
2237    * be for a table/column family/qualifier.
2238    * <p>
2239    * It's also called by the shell, in case you want to find references.
2240    *
2241    * @param protocol the AccessControlService protocol proxy
2242    * @param userShortName the short name of the user to revoke permissions
2243    * @param tableName optional table name
2244    * @param f optional column family
2245    * @param q optional qualifier
2246    * @param actions the permissions to be revoked
2247    * @throws ServiceException
2248    */
2249   public static void revoke(AccessControlService.BlockingInterface protocol,
2250       String userShortName, TableName tableName, byte[] f, byte[] q,
2251       Permission.Action... actions) throws ServiceException {
2252     List<AccessControlProtos.Permission.Action> permActions =
2253         Lists.newArrayListWithCapacity(actions.length);
2254     for (Permission.Action a : actions) {
2255       permActions.add(ProtobufUtil.toPermissionAction(a));
2256     }
2257     AccessControlProtos.RevokeRequest request = RequestConverter.
2258       buildRevokeRequest(userShortName, tableName, f, q, permActions.toArray(
2259         new AccessControlProtos.Permission.Action[actions.length]));
2260     protocol.revoke(null, request);
2261   }
2262 
2263   /**
2264    * A utility used to revoke a user's namespace permissions.
2265    * <p>
2266    * It's also called by the shell, in case you want to find references.
2267    *
2268    * @param protocol the AccessControlService protocol proxy
2269    * @param userShortName the short name of the user to revoke permissions
2270    * @param namespace optional table name
2271    * @param actions the permissions to be revoked
2272    * @throws ServiceException
2273    */
2274   public static void revoke(AccessControlService.BlockingInterface protocol,
2275       String userShortName, String namespace,
2276       Permission.Action... actions) throws ServiceException {
2277     List<AccessControlProtos.Permission.Action> permActions =
2278         Lists.newArrayListWithCapacity(actions.length);
2279     for (Permission.Action a : actions) {
2280       permActions.add(ProtobufUtil.toPermissionAction(a));
2281     }
2282     AccessControlProtos.RevokeRequest request = RequestConverter.
2283       buildRevokeRequest(userShortName, namespace, permActions.toArray(
2284         new AccessControlProtos.Permission.Action[actions.length]));
2285     protocol.revoke(null, request);
2286   }
2287 
2288   /**
2289    * A utility used to get user's global permissions.
2290    * <p>
2291    * It's also called by the shell, in case you want to find references.
2292    *
2293    * @param protocol the AccessControlService protocol proxy
2294    * @throws ServiceException
2295    */
2296   public static List<UserPermission> getUserPermissions(
2297       AccessControlService.BlockingInterface protocol) throws ServiceException {
2298     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2299       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2300     builder.setType(AccessControlProtos.Permission.Type.Global);
2301     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2302     AccessControlProtos.GetUserPermissionsResponse response =
2303       protocol.getUserPermissions(null, request);
2304     List<UserPermission> perms = new ArrayList<UserPermission>(response.getUserPermissionCount());
2305     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2306       perms.add(ProtobufUtil.toUserPermission(perm));
2307     }
2308     return perms;
2309   }
2310 
2311   /**
2312    * A utility used to get user table permissions.
2313    * <p>
2314    * It's also called by the shell, in case you want to find references.
2315    *
2316    * @param protocol the AccessControlService protocol proxy
2317    * @param t optional table name
2318    * @throws ServiceException
2319    */
2320   public static List<UserPermission> getUserPermissions(
2321       AccessControlService.BlockingInterface protocol,
2322       TableName t) throws ServiceException {
2323     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2324       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2325     if (t != null) {
2326       builder.setTableName(ProtobufUtil.toProtoTableName(t));
2327     }
2328     builder.setType(AccessControlProtos.Permission.Type.Table);
2329     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2330     AccessControlProtos.GetUserPermissionsResponse response =
2331       protocol.getUserPermissions(null, request);
2332     List<UserPermission> perms = new ArrayList<UserPermission>(response.getUserPermissionCount());
2333     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2334       perms.add(ProtobufUtil.toUserPermission(perm));
2335     }
2336     return perms;
2337   }
2338 
2339   /**
2340    * A utility used to get permissions for selected namespace.
2341    * <p>
2342    * It's also called by the shell, in case you want to find references.
2343    *
2344    * @param protocol the AccessControlService protocol proxy
2345    * @param namespace name of the namespace
2346    * @throws ServiceException
2347    */
2348   public static List<UserPermission> getUserPermissions(
2349       AccessControlService.BlockingInterface protocol,
2350       byte[] namespace) throws ServiceException {
2351     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2352       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2353     if (namespace != null) {
2354       builder.setNamespaceName(ByteStringer.wrap(namespace));
2355     }
2356     builder.setType(AccessControlProtos.Permission.Type.Namespace);
2357     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2358     AccessControlProtos.GetUserPermissionsResponse response =
2359       protocol.getUserPermissions(null, request);
2360     List<UserPermission> perms = new ArrayList<UserPermission>(response.getUserPermissionCount());
2361     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2362       perms.add(ProtobufUtil.toUserPermission(perm));
2363     }
2364     return perms;
2365   }
2366 
2367   /**
2368    * Convert a protobuf UserTablePermissions to a
2369    * ListMultimap&lt;String, TablePermission&gt; where key is username.
2370    *
2371    * @param proto the protobuf UserPermission
2372    * @return the converted UserPermission
2373    */
2374   public static ListMultimap<String, TablePermission> toUserTablePermissions(
2375       AccessControlProtos.UsersAndPermissions proto) {
2376     ListMultimap<String, TablePermission> perms = ArrayListMultimap.create();
2377     AccessControlProtos.UsersAndPermissions.UserPermissions userPerm;
2378 
2379     for (int i = 0; i < proto.getUserPermissionsCount(); i++) {
2380       userPerm = proto.getUserPermissions(i);
2381       for (int j = 0; j < userPerm.getPermissionsCount(); j++) {
2382         TablePermission tablePerm = toTablePermission(userPerm.getPermissions(j));
2383         perms.put(userPerm.getUser().toStringUtf8(), tablePerm);
2384       }
2385     }
2386 
2387     return perms;
2388   }
2389 
2390   /**
2391    * Converts a Token instance (with embedded identifier) to the protobuf representation.
2392    *
2393    * @param token the Token instance to copy
2394    * @return the protobuf Token message
2395    */
2396   public static AuthenticationProtos.Token toToken(Token<AuthenticationTokenIdentifier> token) {
2397     AuthenticationProtos.Token.Builder builder = AuthenticationProtos.Token.newBuilder();
2398     builder.setIdentifier(ByteStringer.wrap(token.getIdentifier()));
2399     builder.setPassword(ByteStringer.wrap(token.getPassword()));
2400     if (token.getService() != null) {
2401       builder.setService(ByteString.copyFromUtf8(token.getService().toString()));
2402     }
2403     return builder.build();
2404   }
2405 
2406   /**
2407    * Converts a protobuf Token message back into a Token instance.
2408    *
2409    * @param proto the protobuf Token message
2410    * @return the Token instance
2411    */
2412   public static Token<AuthenticationTokenIdentifier> toToken(AuthenticationProtos.Token proto) {
2413     return new Token<AuthenticationTokenIdentifier>(
2414         proto.hasIdentifier() ? proto.getIdentifier().toByteArray() : null,
2415         proto.hasPassword() ? proto.getPassword().toByteArray() : null,
2416         AuthenticationTokenIdentifier.AUTH_TOKEN_TYPE,
2417         proto.hasService() ? new Text(proto.getService().toStringUtf8()) : null);
2418   }
2419 
2420   /**
2421    * Find the HRegion encoded name based on a region specifier
2422    *
2423    * @param regionSpecifier the region specifier
2424    * @return the corresponding region's encoded name
2425    * @throws DoNotRetryIOException if the specifier type is unsupported
2426    */
2427   public static String getRegionEncodedName(
2428       final RegionSpecifier regionSpecifier) throws DoNotRetryIOException {
2429     ByteString value = regionSpecifier.getValue();
2430     RegionSpecifierType type = regionSpecifier.getType();
2431     switch (type) {
2432       case REGION_NAME:
2433         return HRegionInfo.encodeRegionName(value.toByteArray());
2434       case ENCODED_REGION_NAME:
2435         return value.toStringUtf8();
2436       default:
2437         throw new DoNotRetryIOException(
2438           "Unsupported region specifier type: " + type);
2439     }
2440   }
2441 
2442   public static ScanMetrics toScanMetrics(final byte[] bytes) {
2443     Parser<MapReduceProtos.ScanMetrics> parser = MapReduceProtos.ScanMetrics.PARSER;
2444     MapReduceProtos.ScanMetrics pScanMetrics = null;
2445     try {
2446       pScanMetrics = parser.parseFrom(bytes);
2447     } catch (InvalidProtocolBufferException e) {
2448       //Ignored there are just no key values to add.
2449     }
2450     ScanMetrics scanMetrics = new ScanMetrics();
2451     if (pScanMetrics != null) {
2452       for (HBaseProtos.NameInt64Pair pair : pScanMetrics.getMetricsList()) {
2453         if (pair.hasName() && pair.hasValue()) {
2454           scanMetrics.setCounter(pair.getName(), pair.getValue());
2455         }
2456       }
2457     }
2458     return scanMetrics;
2459   }
2460 
2461   public static MapReduceProtos.ScanMetrics toScanMetrics(ScanMetrics scanMetrics) {
2462     MapReduceProtos.ScanMetrics.Builder builder = MapReduceProtos.ScanMetrics.newBuilder();
2463     Map<String, Long> metrics = scanMetrics.getMetricsMap();
2464     for (Entry<String, Long> e : metrics.entrySet()) {
2465       HBaseProtos.NameInt64Pair nameInt64Pair =
2466           HBaseProtos.NameInt64Pair.newBuilder()
2467               .setName(e.getKey())
2468               .setValue(e.getValue())
2469               .build();
2470       builder.addMetrics(nameInt64Pair);
2471     }
2472     return builder.build();
2473   }
2474 
2475   /**
2476    * Unwraps an exception from a protobuf service into the underlying (expected) IOException.
2477    * This method will <strong>always</strong> throw an exception.
2478    * @param se the {@code ServiceException} instance to convert into an {@code IOException}
2479    */
2480   public static void toIOException(ServiceException se) throws IOException {
2481     if (se == null) {
2482       throw new NullPointerException("Null service exception passed!");
2483     }
2484 
2485     Throwable cause = se.getCause();
2486     if (cause != null && cause instanceof IOException) {
2487       throw (IOException)cause;
2488     }
2489     throw new IOException(se);
2490   }
2491 
2492   public static CellProtos.Cell toCell(final Cell kv) {
2493     // Doing this is going to kill us if we do it for all data passed.
2494     // St.Ack 20121205
2495     CellProtos.Cell.Builder kvbuilder = CellProtos.Cell.newBuilder();
2496     kvbuilder.setRow(ByteStringer.wrap(kv.getRowArray(), kv.getRowOffset(),
2497         kv.getRowLength()));
2498     kvbuilder.setFamily(ByteStringer.wrap(kv.getFamilyArray(),
2499         kv.getFamilyOffset(), kv.getFamilyLength()));
2500     kvbuilder.setQualifier(ByteStringer.wrap(kv.getQualifierArray(),
2501         kv.getQualifierOffset(), kv.getQualifierLength()));
2502     kvbuilder.setCellType(CellProtos.CellType.valueOf(kv.getTypeByte()));
2503     kvbuilder.setTimestamp(kv.getTimestamp());
2504     kvbuilder.setValue(ByteStringer.wrap(kv.getValueArray(), kv.getValueOffset(),
2505         kv.getValueLength()));
2506     return kvbuilder.build();
2507   }
2508 
2509   public static Cell toCell(final CellProtos.Cell cell) {
2510     // Doing this is going to kill us if we do it for all data passed.
2511     // St.Ack 20121205
2512     return CellUtil.createCell(cell.getRow().toByteArray(),
2513       cell.getFamily().toByteArray(),
2514       cell.getQualifier().toByteArray(),
2515       cell.getTimestamp(),
2516       (byte)cell.getCellType().getNumber(),
2517       cell.getValue().toByteArray());
2518   }
2519 
2520   public static HBaseProtos.NamespaceDescriptor toProtoNamespaceDescriptor(NamespaceDescriptor ns) {
2521     HBaseProtos.NamespaceDescriptor.Builder b =
2522         HBaseProtos.NamespaceDescriptor.newBuilder()
2523             .setName(ByteString.copyFromUtf8(ns.getName()));
2524     for(Map.Entry<String, String> entry: ns.getConfiguration().entrySet()) {
2525       b.addConfiguration(HBaseProtos.NameStringPair.newBuilder()
2526           .setName(entry.getKey())
2527           .setValue(entry.getValue()));
2528     }
2529     return b.build();
2530   }
2531 
2532   public static NamespaceDescriptor toNamespaceDescriptor(
2533       HBaseProtos.NamespaceDescriptor desc) throws IOException {
2534     NamespaceDescriptor.Builder b =
2535       NamespaceDescriptor.create(desc.getName().toStringUtf8());
2536     for(HBaseProtos.NameStringPair prop : desc.getConfigurationList()) {
2537       b.addConfiguration(prop.getName(), prop.getValue());
2538     }
2539     return b.build();
2540   }
2541 
2542   /**
2543    * Get an instance of the argument type declared in a class's signature. The
2544    * argument type is assumed to be a PB Message subclass, and the instance is
2545    * created using parseFrom method on the passed ByteString.
2546    * @param runtimeClass the runtime type of the class
2547    * @param position the position of the argument in the class declaration
2548    * @param b the ByteString which should be parsed to get the instance created
2549    * @return the instance
2550    * @throws IOException
2551    */
2552   @SuppressWarnings("unchecked")
2553   public static <T extends Message>
2554   T getParsedGenericInstance(Class<?> runtimeClass, int position, ByteString b)
2555       throws IOException {
2556     Type type = runtimeClass.getGenericSuperclass();
2557     Type argType = ((ParameterizedType)type).getActualTypeArguments()[position];
2558     Class<T> classType = (Class<T>)argType;
2559     T inst;
2560     try {
2561       Method m = classType.getMethod("parseFrom", ByteString.class);
2562       inst = (T)m.invoke(null, b);
2563       return inst;
2564     } catch (SecurityException e) {
2565       throw new IOException(e);
2566     } catch (NoSuchMethodException e) {
2567       throw new IOException(e);
2568     } catch (IllegalArgumentException e) {
2569       throw new IOException(e);
2570     } catch (InvocationTargetException e) {
2571       throw new IOException(e);
2572     } catch (IllegalAccessException e) {
2573       throw new IOException(e);
2574     }
2575   }
2576 
2577   @SuppressWarnings("deprecation")
2578   public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family,
2579       List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
2580     return toCompactionDescriptor(info, null, family, inputPaths, outputPaths, storeDir);
2581   }
2582 
2583   @SuppressWarnings("deprecation")
2584   public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] regionName,
2585       byte[] family, List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
2586     // compaction descriptor contains relative paths.
2587     // input / output paths are relative to the store dir
2588     // store dir is relative to region dir
2589     CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder()
2590         .setTableName(ByteStringer.wrap(info.getTable().toBytes()))
2591         .setEncodedRegionName(ByteStringer.wrap(
2592           regionName == null ? info.getEncodedNameAsBytes() : regionName))
2593         .setFamilyName(ByteStringer.wrap(family))
2594         .setStoreHomeDir(storeDir.getName()); //make relative
2595     for (Path inputPath : inputPaths) {
2596       builder.addCompactionInput(inputPath.getName()); //relative path
2597     }
2598     for (Path outputPath : outputPaths) {
2599       builder.addCompactionOutput(outputPath.getName());
2600     }
2601     builder.setRegionName(ByteStringer.wrap(info.getRegionName()));
2602     return builder.build();
2603   }
2604 
2605   public static FlushDescriptor toFlushDescriptor(FlushAction action, HRegionInfo hri,
2606       long flushSeqId, Map<byte[], List<Path>> committedFiles) {
2607     FlushDescriptor.Builder desc = FlushDescriptor.newBuilder()
2608         .setAction(action)
2609         .setEncodedRegionName(ByteStringer.wrap(hri.getEncodedNameAsBytes()))
2610         .setRegionName(ByteStringer.wrap(hri.getRegionName()))
2611         .setFlushSequenceNumber(flushSeqId)
2612         .setTableName(ByteStringer.wrap(hri.getTable().getName()));
2613 
2614     for (Map.Entry<byte[], List<Path>> entry : committedFiles.entrySet()) {
2615       WALProtos.FlushDescriptor.StoreFlushDescriptor.Builder builder =
2616           WALProtos.FlushDescriptor.StoreFlushDescriptor.newBuilder()
2617           .setFamilyName(ByteStringer.wrap(entry.getKey()))
2618           .setStoreHomeDir(Bytes.toString(entry.getKey())); //relative to region
2619       if (entry.getValue() != null) {
2620         for (Path path : entry.getValue()) {
2621           builder.addFlushOutput(path.getName());
2622         }
2623       }
2624       desc.addStoreFlushes(builder);
2625     }
2626     return desc.build();
2627   }
2628 
2629   public static RegionEventDescriptor toRegionEventDescriptor(
2630       EventType eventType, HRegionInfo hri, long seqId, ServerName server,
2631       Map<byte[], List<Path>> storeFiles) {
2632     RegionEventDescriptor.Builder desc = RegionEventDescriptor.newBuilder()
2633         .setEventType(eventType)
2634         .setTableName(ByteStringer.wrap(hri.getTable().getName()))
2635         .setEncodedRegionName(ByteStringer.wrap(hri.getEncodedNameAsBytes()))
2636         .setRegionName(ByteStringer.wrap(hri.getRegionName()))
2637         .setLogSequenceNumber(seqId)
2638         .setServer(toServerName(server));
2639 
2640     for (Map.Entry<byte[], List<Path>> entry : storeFiles.entrySet()) {
2641       StoreDescriptor.Builder builder = StoreDescriptor.newBuilder()
2642           .setFamilyName(ByteStringer.wrap(entry.getKey()))
2643           .setStoreHomeDir(Bytes.toString(entry.getKey()));
2644       for (Path path : entry.getValue()) {
2645         builder.addStoreFile(path.getName());
2646       }
2647 
2648       desc.addStores(builder);
2649     }
2650     return desc.build();
2651   }
2652 
2653   /**
2654    * Return short version of Message toString'd, shorter than TextFormat#shortDebugString.
2655    * Tries to NOT print out data both because it can be big but also so we do not have data in our
2656    * logs. Use judiciously.
2657    * @param m
2658    * @return toString of passed <code>m</code>
2659    */
2660   public static String getShortTextFormat(Message m) {
2661     if (m == null) return "null";
2662     if (m instanceof ScanRequest) {
2663       // This should be small and safe to output.  No data.
2664       return TextFormat.shortDebugString(m);
2665     } else if (m instanceof RegionServerReportRequest) {
2666       // Print a short message only, just the servername and the requests, not the full load.
2667       RegionServerReportRequest r = (RegionServerReportRequest)m;
2668       return "server " + TextFormat.shortDebugString(r.getServer()) +
2669         " load { numberOfRequests: " + r.getLoad().getNumberOfRequests() + " }";
2670     } else if (m instanceof RegionServerStartupRequest) {
2671       // Should be small enough.
2672       return TextFormat.shortDebugString(m);
2673     } else if (m instanceof MutationProto) {
2674       return toShortString((MutationProto)m);
2675     } else if (m instanceof GetRequest) {
2676       GetRequest r = (GetRequest) m;
2677       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2678           ", row=" + getStringForByteString(r.getGet().getRow());
2679     } else if (m instanceof ClientProtos.MultiRequest) {
2680       ClientProtos.MultiRequest r = (ClientProtos.MultiRequest) m;
2681       // Get first set of Actions.
2682       ClientProtos.RegionAction actions = r.getRegionActionList().get(0);
2683       String row = actions.getActionCount() <= 0? "":
2684         getStringForByteString(actions.getAction(0).hasGet()?
2685           actions.getAction(0).getGet().getRow():
2686           actions.getAction(0).getMutation().getRow());
2687       return "region= " + getStringForByteString(actions.getRegion().getValue()) +
2688           ", for " + r.getRegionActionCount() +
2689           " actions and 1st row key=" + row;
2690     } else if (m instanceof ClientProtos.MutateRequest) {
2691       ClientProtos.MutateRequest r = (ClientProtos.MutateRequest) m;
2692       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2693           ", row=" + getStringForByteString(r.getMutation().getRow());
2694     }
2695     return "TODO: " + m.getClass().toString();
2696   }
2697 
2698   private static String getStringForByteString(ByteString bs) {
2699     return Bytes.toStringBinary(bs.toByteArray());
2700   }
2701 
2702   /**
2703    * Print out some subset of a MutationProto rather than all of it and its data
2704    * @param proto Protobuf to print out
2705    * @return Short String of mutation proto
2706    */
2707   static String toShortString(final MutationProto proto) {
2708     return "row=" + Bytes.toString(proto.getRow().toByteArray()) +
2709         ", type=" + proto.getMutateType().toString();
2710   }
2711 
2712   public static TableName toTableName(HBaseProtos.TableName tableNamePB) {
2713     return TableName.valueOf(tableNamePB.getNamespace().asReadOnlyByteBuffer(),
2714         tableNamePB.getQualifier().asReadOnlyByteBuffer());
2715   }
2716 
2717   public static HBaseProtos.TableName toProtoTableName(TableName tableName) {
2718     return HBaseProtos.TableName.newBuilder()
2719         .setNamespace(ByteStringer.wrap(tableName.getNamespace()))
2720         .setQualifier(ByteStringer.wrap(tableName.getQualifier())).build();
2721   }
2722 
2723   public static TableName[] getTableNameArray(List<HBaseProtos.TableName> tableNamesList) {
2724     if (tableNamesList == null) {
2725       return new TableName[0];
2726     }
2727     TableName[] tableNames = new TableName[tableNamesList.size()];
2728     for (int i = 0; i < tableNamesList.size(); i++) {
2729       tableNames[i] = toTableName(tableNamesList.get(i));
2730     }
2731     return tableNames;
2732   }
2733 
2734   /**
2735    * Convert a protocol buffer CellVisibility to a client CellVisibility
2736    *
2737    * @param proto
2738    * @return the converted client CellVisibility
2739    */
2740   public static CellVisibility toCellVisibility(ClientProtos.CellVisibility proto) {
2741     if (proto == null) return null;
2742     return new CellVisibility(proto.getExpression());
2743   }
2744 
2745   /**
2746    * Convert a protocol buffer CellVisibility bytes to a client CellVisibility
2747    *
2748    * @param protoBytes
2749    * @return the converted client CellVisibility
2750    * @throws DeserializationException
2751    */
2752   public static CellVisibility toCellVisibility(byte[] protoBytes) throws DeserializationException {
2753     if (protoBytes == null) return null;
2754     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2755     ClientProtos.CellVisibility proto = null;
2756     try {
2757       ProtobufUtil.mergeFrom(builder, protoBytes);
2758       proto = builder.build();
2759     } catch (IOException e) {
2760       throw new DeserializationException(e);
2761     }
2762     return toCellVisibility(proto);
2763   }
2764 
2765   /**
2766    * Create a protocol buffer CellVisibility based on a client CellVisibility.
2767    *
2768    * @param cellVisibility
2769    * @return a protocol buffer CellVisibility
2770    */
2771   public static ClientProtos.CellVisibility toCellVisibility(CellVisibility cellVisibility) {
2772     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2773     builder.setExpression(cellVisibility.getExpression());
2774     return builder.build();
2775   }
2776 
2777   /**
2778    * Convert a protocol buffer Authorizations to a client Authorizations
2779    *
2780    * @param proto
2781    * @return the converted client Authorizations
2782    */
2783   public static Authorizations toAuthorizations(ClientProtos.Authorizations proto) {
2784     if (proto == null) return null;
2785     return new Authorizations(proto.getLabelList());
2786   }
2787 
2788   /**
2789    * Convert a protocol buffer Authorizations bytes to a client Authorizations
2790    *
2791    * @param protoBytes
2792    * @return the converted client Authorizations
2793    * @throws DeserializationException
2794    */
2795   public static Authorizations toAuthorizations(byte[] protoBytes) throws DeserializationException {
2796     if (protoBytes == null) return null;
2797     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2798     ClientProtos.Authorizations proto = null;
2799     try {
2800       ProtobufUtil.mergeFrom(builder, protoBytes);
2801       proto = builder.build();
2802     } catch (IOException e) {
2803       throw new DeserializationException(e);
2804     }
2805     return toAuthorizations(proto);
2806   }
2807 
2808   /**
2809    * Create a protocol buffer Authorizations based on a client Authorizations.
2810    *
2811    * @param authorizations
2812    * @return a protocol buffer Authorizations
2813    */
2814   public static ClientProtos.Authorizations toAuthorizations(Authorizations authorizations) {
2815     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2816     for (String label : authorizations.getLabels()) {
2817       builder.addLabel(label);
2818     }
2819     return builder.build();
2820   }
2821 
2822   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(String user,
2823       Permission perms) {
2824     return AccessControlProtos.UsersAndPermissions.newBuilder()
2825       .addUserPermissions(AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder()
2826         .setUser(ByteString.copyFromUtf8(user))
2827         .addPermissions(toPermission(perms))
2828         .build())
2829       .build();
2830   }
2831 
2832   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(
2833       ListMultimap<String, Permission> perms) {
2834     AccessControlProtos.UsersAndPermissions.Builder builder =
2835         AccessControlProtos.UsersAndPermissions.newBuilder();
2836     for (Map.Entry<String, Collection<Permission>> entry : perms.asMap().entrySet()) {
2837       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
2838         AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
2839       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
2840       for (Permission perm: entry.getValue()) {
2841         userPermBuilder.addPermissions(toPermission(perm));
2842       }
2843       builder.addUserPermissions(userPermBuilder.build());
2844     }
2845     return builder.build();
2846   }
2847 
2848   public static ListMultimap<String, Permission> toUsersAndPermissions(
2849       AccessControlProtos.UsersAndPermissions proto) {
2850     ListMultimap<String, Permission> result = ArrayListMultimap.create();
2851     for (AccessControlProtos.UsersAndPermissions.UserPermissions userPerms:
2852         proto.getUserPermissionsList()) {
2853       String user = userPerms.getUser().toStringUtf8();
2854       for (AccessControlProtos.Permission perm: userPerms.getPermissionsList()) {
2855         result.put(user, toPermission(perm));
2856       }
2857     }
2858     return result;
2859   }
2860 
2861   /**
2862    * Convert a protocol buffer TimeUnit to a client TimeUnit
2863    *
2864    * @param proto
2865    * @return the converted client TimeUnit
2866    */
2867   public static TimeUnit toTimeUnit(final HBaseProtos.TimeUnit proto) {
2868     switch (proto) {
2869       case NANOSECONDS:  return TimeUnit.NANOSECONDS;
2870       case MICROSECONDS: return TimeUnit.MICROSECONDS;
2871       case MILLISECONDS: return TimeUnit.MILLISECONDS;
2872       case SECONDS:      return TimeUnit.SECONDS;
2873       case MINUTES:      return TimeUnit.MINUTES;
2874       case HOURS:        return TimeUnit.HOURS;
2875       case DAYS:         return TimeUnit.DAYS;
2876     }
2877     throw new RuntimeException("Invalid TimeUnit " + proto);
2878   }
2879 
2880   /**
2881    * Convert a client TimeUnit to a protocol buffer TimeUnit
2882    *
2883    * @param timeUnit
2884    * @return the converted protocol buffer TimeUnit
2885    */
2886   public static HBaseProtos.TimeUnit toProtoTimeUnit(final TimeUnit timeUnit) {
2887     switch (timeUnit) {
2888       case NANOSECONDS:  return HBaseProtos.TimeUnit.NANOSECONDS;
2889       case MICROSECONDS: return HBaseProtos.TimeUnit.MICROSECONDS;
2890       case MILLISECONDS: return HBaseProtos.TimeUnit.MILLISECONDS;
2891       case SECONDS:      return HBaseProtos.TimeUnit.SECONDS;
2892       case MINUTES:      return HBaseProtos.TimeUnit.MINUTES;
2893       case HOURS:        return HBaseProtos.TimeUnit.HOURS;
2894       case DAYS:         return HBaseProtos.TimeUnit.DAYS;
2895     }
2896     throw new RuntimeException("Invalid TimeUnit " + timeUnit);
2897   }
2898 
2899   /**
2900    * Convert a protocol buffer ThrottleType to a client ThrottleType
2901    *
2902    * @param proto
2903    * @return the converted client ThrottleType
2904    */
2905   public static ThrottleType toThrottleType(final QuotaProtos.ThrottleType proto) {
2906     switch (proto) {
2907       case REQUEST_NUMBER: return ThrottleType.REQUEST_NUMBER;
2908       case REQUEST_SIZE:   return ThrottleType.REQUEST_SIZE;
2909       case WRITE_NUMBER:   return ThrottleType.WRITE_NUMBER;
2910       case WRITE_SIZE:     return ThrottleType.WRITE_SIZE;
2911       case READ_NUMBER:    return ThrottleType.READ_NUMBER;
2912       case READ_SIZE:      return ThrottleType.READ_SIZE;
2913     }
2914     throw new RuntimeException("Invalid ThrottleType " + proto);
2915   }
2916 
2917   /**
2918    * Convert a client ThrottleType to a protocol buffer ThrottleType
2919    *
2920    * @param type
2921    * @return the converted protocol buffer ThrottleType
2922    */
2923   public static QuotaProtos.ThrottleType toProtoThrottleType(final ThrottleType type) {
2924     switch (type) {
2925       case REQUEST_NUMBER: return QuotaProtos.ThrottleType.REQUEST_NUMBER;
2926       case REQUEST_SIZE:   return QuotaProtos.ThrottleType.REQUEST_SIZE;
2927       case WRITE_NUMBER:   return QuotaProtos.ThrottleType.WRITE_NUMBER;
2928       case WRITE_SIZE:     return QuotaProtos.ThrottleType.WRITE_SIZE;
2929       case READ_NUMBER:    return QuotaProtos.ThrottleType.READ_NUMBER;
2930       case READ_SIZE:      return QuotaProtos.ThrottleType.READ_SIZE;
2931     }
2932     throw new RuntimeException("Invalid ThrottleType " + type);
2933   }
2934 
2935   /**
2936    * Convert a protocol buffer QuotaScope to a client QuotaScope
2937    *
2938    * @param proto
2939    * @return the converted client QuotaScope
2940    */
2941   public static QuotaScope toQuotaScope(final QuotaProtos.QuotaScope proto) {
2942     switch (proto) {
2943       case CLUSTER: return QuotaScope.CLUSTER;
2944       case MACHINE: return QuotaScope.MACHINE;
2945     }
2946     throw new RuntimeException("Invalid QuotaScope " + proto);
2947   }
2948 
2949   /**
2950    * Convert a client QuotaScope to a protocol buffer QuotaScope
2951    *
2952    * @param scope
2953    * @return the converted protocol buffer QuotaScope
2954    */
2955   public static QuotaProtos.QuotaScope toProtoQuotaScope(final QuotaScope scope) {
2956     switch (scope) {
2957       case CLUSTER: return QuotaProtos.QuotaScope.CLUSTER;
2958       case MACHINE: return QuotaProtos.QuotaScope.MACHINE;
2959     }
2960     throw new RuntimeException("Invalid QuotaScope " + scope);
2961   }
2962 
2963   /**
2964    * Convert a protocol buffer QuotaType to a client QuotaType
2965    *
2966    * @param proto
2967    * @return the converted client QuotaType
2968    */
2969   public static QuotaType toQuotaScope(final QuotaProtos.QuotaType proto) {
2970     switch (proto) {
2971       case THROTTLE: return QuotaType.THROTTLE;
2972     }
2973     throw new RuntimeException("Invalid QuotaType " + proto);
2974   }
2975 
2976   /**
2977    * Convert a client QuotaType to a protocol buffer QuotaType
2978    *
2979    * @param type
2980    * @return the converted protocol buffer QuotaType
2981    */
2982   public static QuotaProtos.QuotaType toProtoQuotaScope(final QuotaType type) {
2983     switch (type) {
2984       case THROTTLE: return QuotaProtos.QuotaType.THROTTLE;
2985     }
2986     throw new RuntimeException("Invalid QuotaType " + type);
2987   }
2988 
2989   /**
2990    * Build a protocol buffer TimedQuota
2991    *
2992    * @param limit the allowed number of request/data per timeUnit
2993    * @param timeUnit the limit time unit
2994    * @param scope the quota scope
2995    * @return the protocol buffer TimedQuota
2996    */
2997   public static QuotaProtos.TimedQuota toTimedQuota(final long limit, final TimeUnit timeUnit,
2998       final QuotaScope scope) {
2999     return QuotaProtos.TimedQuota.newBuilder()
3000             .setSoftLimit(limit)
3001             .setTimeUnit(toProtoTimeUnit(timeUnit))
3002             .setScope(toProtoQuotaScope(scope))
3003             .build();
3004   }
3005 
3006   /**
3007    * Generates a marker for the WAL so that we propagate the notion of a bulk region load
3008    * throughout the WAL.
3009    *
3010    * @param tableName         The tableName into which the bulk load is being imported into.
3011    * @param encodedRegionName Encoded region name of the region which is being bulk loaded.
3012    * @param storeFiles        A set of store files of a column family are bulk loaded.
3013    * @param bulkloadSeqId     sequence ID (by a force flush) used to create bulk load hfile
3014    *                          name
3015    * @return The WAL log marker for bulk loads.
3016    */
3017   public static WALProtos.BulkLoadDescriptor toBulkLoadDescriptor(TableName tableName,
3018       ByteString encodedRegionName, Map<byte[], List<Path>> storeFiles, long bulkloadSeqId) {
3019     BulkLoadDescriptor.Builder desc = BulkLoadDescriptor.newBuilder()
3020         .setTableName(ProtobufUtil.toProtoTableName(tableName))
3021         .setEncodedRegionName(encodedRegionName).setBulkloadSeqNum(bulkloadSeqId);
3022 
3023     for (Map.Entry<byte[], List<Path>> entry : storeFiles.entrySet()) {
3024       WALProtos.StoreDescriptor.Builder builder = StoreDescriptor.newBuilder()
3025           .setFamilyName(ByteStringer.wrap(entry.getKey()))
3026           .setStoreHomeDir(Bytes.toString(entry.getKey())); // relative to region
3027       for (Path path : entry.getValue()) {
3028         builder.addStoreFile(path.getName());
3029       }
3030       desc.addStores(builder);
3031     }
3032 
3033     return desc.build();
3034   }
3035 
3036   /**
3037    * This version of protobuf's mergeDelimitedFrom avoid the hard-coded 64MB limit for decoding
3038    * buffers
3039    * @param builder current message builder
3040    * @param in Inputsream with delimited protobuf data
3041    * @throws IOException
3042    */
3043   public static void mergeDelimitedFrom(Message.Builder builder, InputStream in)
3044     throws IOException {
3045     // This used to be builder.mergeDelimitedFrom(in);
3046     // but is replaced to allow us to bump the protobuf size limit.
3047     final int firstByte = in.read();
3048     if (firstByte != -1) {
3049       final int size = CodedInputStream.readRawVarint32(firstByte, in);
3050       final InputStream limitedInput = new LimitInputStream(in, size);
3051       final CodedInputStream codedInput = CodedInputStream.newInstance(limitedInput);
3052       codedInput.setSizeLimit(size);
3053       builder.mergeFrom(codedInput);
3054       codedInput.checkLastTagWas(0);
3055     }
3056   }
3057 
3058   /**
3059    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3060    * buffers where the message size is known
3061    * @param builder current message builder
3062    * @param in InputStream containing protobuf data
3063    * @param size known size of protobuf data
3064    * @throws IOException 
3065    */
3066   public static void mergeFrom(Message.Builder builder, InputStream in, int size)
3067       throws IOException {
3068     final CodedInputStream codedInput = CodedInputStream.newInstance(in);
3069     codedInput.setSizeLimit(size);
3070     builder.mergeFrom(codedInput);
3071     codedInput.checkLastTagWas(0);
3072   }
3073 
3074   /**
3075    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3076    * buffers where the message size is not known
3077    * @param builder current message builder
3078    * @param in InputStream containing protobuf data
3079    * @throws IOException 
3080    */
3081   public static void mergeFrom(Message.Builder builder, InputStream in)
3082       throws IOException {
3083     final CodedInputStream codedInput = CodedInputStream.newInstance(in);
3084     codedInput.setSizeLimit(Integer.MAX_VALUE);
3085     builder.mergeFrom(codedInput);
3086     codedInput.checkLastTagWas(0);
3087   }
3088 
3089   /**
3090    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3091    * buffers when working with ByteStrings
3092    * @param builder current message builder
3093    * @param bs ByteString containing the 
3094    * @throws IOException 
3095    */
3096   public static void mergeFrom(Message.Builder builder, ByteString bs) throws IOException {
3097     final CodedInputStream codedInput = bs.newCodedInput();
3098     codedInput.setSizeLimit(bs.size());
3099     builder.mergeFrom(codedInput);
3100     codedInput.checkLastTagWas(0);
3101   }
3102 
3103   /**
3104    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3105    * buffers when working with byte arrays
3106    * @param builder current message builder
3107    * @param b byte array
3108    * @throws IOException 
3109    */
3110   public static void mergeFrom(Message.Builder builder, byte[] b) throws IOException {
3111     final CodedInputStream codedInput = CodedInputStream.newInstance(b);
3112     codedInput.setSizeLimit(b.length);
3113     builder.mergeFrom(codedInput);
3114     codedInput.checkLastTagWas(0);
3115   }
3116 
3117   /**
3118    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3119    * buffers when working with byte arrays
3120    * @param builder current message builder
3121    * @param b byte array
3122    * @param offset
3123    * @param length
3124    * @throws IOException
3125    */
3126   public static void mergeFrom(Message.Builder builder, byte[] b, int offset, int length)
3127       throws IOException {
3128     final CodedInputStream codedInput = CodedInputStream.newInstance(b, offset, length);
3129     codedInput.setSizeLimit(length);
3130     builder.mergeFrom(codedInput);
3131     codedInput.checkLastTagWas(0);
3132   }
3133 
3134   public static void mergeFrom(Message.Builder builder, CodedInputStream codedInput, int length)
3135       throws IOException {
3136     codedInput.resetSizeCounter();
3137     int prevLimit = codedInput.setSizeLimit(length);
3138 
3139     int limit = codedInput.pushLimit(length);
3140     builder.mergeFrom(codedInput);
3141     codedInput.popLimit(limit);
3142 
3143     codedInput.checkLastTagWas(0);
3144     codedInput.setSizeLimit(prevLimit);
3145   }
3146 
3147   public static ReplicationLoadSink toReplicationLoadSink(
3148       ClusterStatusProtos.ReplicationLoadSink cls) {
3149     return new ReplicationLoadSink(cls.getAgeOfLastAppliedOp(), cls.getTimeStampsOfLastAppliedOp());
3150   }
3151 
3152   public static ReplicationLoadSource toReplicationLoadSource(
3153       ClusterStatusProtos.ReplicationLoadSource cls) {
3154     return new ReplicationLoadSource(cls.getPeerID(), cls.getAgeOfLastShippedOp(),
3155         cls.getSizeOfLogQueue(), cls.getTimeStampOfLastShippedOp(), cls.getReplicationLag());
3156   }
3157 
3158   public static List<ReplicationLoadSource> toReplicationLoadSourceList(
3159       List<ClusterStatusProtos.ReplicationLoadSource> clsList) {
3160     ArrayList<ReplicationLoadSource> rlsList = new ArrayList<ReplicationLoadSource>();
3161     for (ClusterStatusProtos.ReplicationLoadSource cls : clsList) {
3162       rlsList.add(toReplicationLoadSource(cls));
3163     }
3164     return rlsList;
3165   }
3166 
3167   /**
3168    * Get a protocol buffer VersionInfo
3169    *
3170    * @return the converted protocol buffer VersionInfo
3171    */
3172   public static HBaseProtos.VersionInfo getVersionInfo() {
3173     HBaseProtos.VersionInfo.Builder builder = HBaseProtos.VersionInfo.newBuilder();
3174     String version = VersionInfo.getVersion();
3175     builder.setVersion(version);
3176     String[] components = version.split("\\.");
3177     if (components != null && components.length > 2) {
3178       builder.setVersionMajor(Integer.parseInt(components[0]));
3179       builder.setVersionMinor(Integer.parseInt(components[1]));
3180     }
3181     builder.setUrl(VersionInfo.getUrl());
3182     builder.setRevision(VersionInfo.getRevision());
3183     builder.setUser(VersionInfo.getUser());
3184     builder.setDate(VersionInfo.getDate());
3185     builder.setSrcChecksum(VersionInfo.getSrcChecksum());
3186     return builder.build();
3187   }
3188 
3189   /**
3190    * Convert SecurityCapabilitiesResponse.Capability to SecurityCapability
3191    * @param capabilities capabilities returned in the SecurityCapabilitiesResponse message
3192    * @return the converted list of SecurityCapability elements
3193    */
3194   public static List<SecurityCapability> toSecurityCapabilityList(
3195       List<MasterProtos.SecurityCapabilitiesResponse.Capability> capabilities) {
3196     List<SecurityCapability> scList = new ArrayList<>(capabilities.size());
3197     for (MasterProtos.SecurityCapabilitiesResponse.Capability c: capabilities) {
3198       try {
3199         scList.add(SecurityCapability.valueOf(c.getNumber()));
3200       } catch (IllegalArgumentException e) {
3201         // Unknown capability, just ignore it. We don't understand the new capability
3202         // but don't care since by definition we cannot take advantage of it.
3203       }
3204     }
3205     return scList;
3206   }
3207 
3208   private static HBaseProtos.TimeRange.Builder timeRangeToProto(TimeRange timeRange) {
3209     HBaseProtos.TimeRange.Builder timeRangeBuilder =
3210         HBaseProtos.TimeRange.newBuilder();
3211     timeRangeBuilder.setFrom(timeRange.getMin());
3212     timeRangeBuilder.setTo(timeRange.getMax());
3213     return timeRangeBuilder;
3214   }
3215 
3216   private static TimeRange protoToTimeRange(HBaseProtos.TimeRange timeRange) throws IOException {
3217       long minStamp = 0;
3218       long maxStamp = Long.MAX_VALUE;
3219       if (timeRange.hasFrom()) {
3220         minStamp = timeRange.getFrom();
3221       }
3222       if (timeRange.hasTo()) {
3223         maxStamp = timeRange.getTo();
3224       }
3225     return new TimeRange(minStamp, maxStamp);
3226   }
3227 
3228 }