View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.protobuf;
19  
20  import java.io.ByteArrayOutputStream;
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.io.InterruptedIOException;
24  import java.lang.reflect.Constructor;
25  import java.lang.reflect.InvocationTargetException;
26  import java.lang.reflect.Method;
27  import java.lang.reflect.ParameterizedType;
28  import java.lang.reflect.Type;
29  import java.nio.ByteBuffer;
30  import java.security.PrivilegedExceptionAction;
31  import java.util.ArrayList;
32  import java.util.Collection;
33  import java.util.HashMap;
34  import java.util.HashSet;
35  import java.util.List;
36  import java.util.Locale;
37  import java.util.Map;
38  import java.util.Map.Entry;
39  import java.util.NavigableSet;
40  import java.util.Set;
41  import java.util.concurrent.TimeUnit;
42
43  import org.apache.hadoop.conf.Configuration;
44  import org.apache.hadoop.fs.Path;
45
46  import static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier
47  .RegionSpecifierType.REGION_NAME;
48
49  import org.apache.hadoop.hbase.Cell;
50  import org.apache.hadoop.hbase.CellScanner;
51  import org.apache.hadoop.hbase.CellUtil;
52  import org.apache.hadoop.hbase.ClusterId;
53  import org.apache.hadoop.hbase.ClusterStatus;
54  import org.apache.hadoop.hbase.DoNotRetryIOException;
55  import org.apache.hadoop.hbase.HBaseConfiguration;
56  import org.apache.hadoop.hbase.HColumnDescriptor;
57  import org.apache.hadoop.hbase.HConstants;
58  import org.apache.hadoop.hbase.HRegionInfo;
59  import org.apache.hadoop.hbase.HTableDescriptor;
60  import org.apache.hadoop.hbase.KeyValue;
61  import org.apache.hadoop.hbase.NamespaceDescriptor;
62  import org.apache.hadoop.hbase.ServerLoad;
63  import org.apache.hadoop.hbase.ServerName;
64  import org.apache.hadoop.hbase.TableName;
65  import org.apache.hadoop.hbase.Tag;
66  import org.apache.hadoop.hbase.TagUtil;
67  import org.apache.hadoop.hbase.classification.InterfaceAudience;
68  import org.apache.hadoop.hbase.client.Append;
69  import org.apache.hadoop.hbase.client.CompactionState;
70  import org.apache.hadoop.hbase.client.Consistency;
71  import org.apache.hadoop.hbase.client.Delete;
72  import org.apache.hadoop.hbase.client.Durability;
73  import org.apache.hadoop.hbase.client.Get;
74  import org.apache.hadoop.hbase.client.Increment;
75  import org.apache.hadoop.hbase.client.Mutation;
76  import org.apache.hadoop.hbase.client.Put;
77  import org.apache.hadoop.hbase.client.RegionLoadStats;
78  import org.apache.hadoop.hbase.client.Result;
79  import org.apache.hadoop.hbase.client.Scan;
80  import org.apache.hadoop.hbase.client.SnapshotDescription;
81  import org.apache.hadoop.hbase.client.SnapshotType;
82  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
83  import org.apache.hadoop.hbase.client.security.SecurityCapability;
84  import org.apache.hadoop.hbase.exceptions.DeserializationException;
85  import org.apache.hadoop.hbase.filter.ByteArrayComparable;
86  import org.apache.hadoop.hbase.filter.Filter;
87  import org.apache.hadoop.hbase.io.LimitInputStream;
88  import org.apache.hadoop.hbase.io.TimeRange;
89  import org.apache.hadoop.hbase.master.RegionState;
90  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
91  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService;
92  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
93  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
94  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
95  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
96  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
97  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
98  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
99  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
100 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
101 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
102 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
103 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
104 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
105 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
106 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
107 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest;
108 import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
109 import org.apache.hadoop.hbase.protobuf.generated.CellProtos;
110 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
111 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
112 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Column;
113 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall;
114 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
115 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
116 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
117 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
118 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue;
119 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue.QualifierValue;
120 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.DeleteType;
121 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
122 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
123 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
124 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.LiveServerInfo;
125 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionInTransition;
126 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
127 import org.apache.hadoop.hbase.protobuf.generated.FSProtos.HBaseVersionFileContent;
128 import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
129 import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
130 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
131 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair;
132 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ColumnFamilySchema;
133 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
134 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
135 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo;
136 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
137 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
138 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema;
139 import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
140 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos;
141 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
142 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
143 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService;
144 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos;
145 import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos;
146 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
147 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
148 import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
149 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.BulkLoadDescriptor;
150 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
151 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
152 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
153 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
154 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor.EventType;
155 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
156 import org.apache.hadoop.hbase.quotas.QuotaScope;
157 import org.apache.hadoop.hbase.quotas.QuotaType;
158 import org.apache.hadoop.hbase.quotas.ThrottleType;
159 import org.apache.hadoop.hbase.replication.ReplicationLoadSink;
160 import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
161 import org.apache.hadoop.hbase.rsgroup.RSGroupInfo;
162 import org.apache.hadoop.hbase.security.access.Permission;
163 import org.apache.hadoop.hbase.security.access.TablePermission;
164 import org.apache.hadoop.hbase.security.access.UserPermission;
165 import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier;
166 import org.apache.hadoop.hbase.security.visibility.Authorizations;
167 import org.apache.hadoop.hbase.security.visibility.CellVisibility;
168 import org.apache.hadoop.hbase.util.ByteStringer;
169 import org.apache.hadoop.hbase.util.Bytes;
170 import org.apache.hadoop.hbase.util.DynamicClassLoader;
171 import org.apache.hadoop.hbase.util.ExceptionUtil;
172 import org.apache.hadoop.hbase.util.Methods;
173 import org.apache.hadoop.hbase.util.Pair;
174 import org.apache.hadoop.hbase.util.VersionInfo;
175 import org.apache.hadoop.io.Text;
176 import org.apache.hadoop.ipc.RemoteException;
177 import org.apache.hadoop.security.UserGroupInformation;
178 import org.apache.hadoop.security.token.Token;
179
180 import com.google.common.collect.ArrayListMultimap;
181 import com.google.common.collect.ListMultimap;
182 import com.google.common.collect.Lists;
183 import com.google.common.net.HostAndPort;
184 import com.google.protobuf.ByteString;
185 import com.google.protobuf.CodedInputStream;
186 import com.google.protobuf.InvalidProtocolBufferException;
187 import com.google.protobuf.Message;
188 import com.google.protobuf.Parser;
189 import com.google.protobuf.RpcChannel;
190 import com.google.protobuf.RpcController;
191 import com.google.protobuf.Service;
192 import com.google.protobuf.ServiceException;
193 import com.google.protobuf.TextFormat;
194
195 /**
196  * Protobufs utility.
197  */
198 @edu.umd.cs.findbugs.annotations.SuppressWarnings(
199   value="DP_CREATE_CLASSLOADER_INSIDE_DO_PRIVILEGED", justification="None. Address sometime.")
200 @InterfaceAudience.Private // TODO: some clients (Hive, etc) use this class
201 public final class ProtobufUtil {
202
203   private ProtobufUtil() {
204   }
205
206   /**
207    * Primitive type to class mapping.
208    */
209   private final static Map<String, Class<?>>
210     PRIMITIVES = new HashMap<String, Class<?>>();
211
212   /**
213    * Many results are simple: no cell, exists true or false. To save on object creations,
214    *  we reuse them across calls.
215    */
216   private final static Cell[] EMPTY_CELL_ARRAY = new Cell[]{};
217   private final static Result EMPTY_RESULT = Result.create(EMPTY_CELL_ARRAY);
218   final static Result EMPTY_RESULT_EXISTS_TRUE = Result.create(null, true);
219   final static Result EMPTY_RESULT_EXISTS_FALSE = Result.create(null, false);
220   private final static Result EMPTY_RESULT_STALE = Result.create(EMPTY_CELL_ARRAY, null, true);
221   private final static Result EMPTY_RESULT_EXISTS_TRUE_STALE
222     = Result.create((Cell[])null, true, true);
223   private final static Result EMPTY_RESULT_EXISTS_FALSE_STALE
224     = Result.create((Cell[])null, false, true);
225
226   private final static ClientProtos.Result EMPTY_RESULT_PB;
227   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_TRUE;
228   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_FALSE;
229   private final static ClientProtos.Result EMPTY_RESULT_PB_STALE;
230   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_TRUE_STALE;
231   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_FALSE_STALE;
232
233
234   static {
235     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
236
237     builder.setExists(true);
238     builder.setAssociatedCellCount(0);
239     EMPTY_RESULT_PB_EXISTS_TRUE =  builder.build();
240
241     builder.setStale(true);
242     EMPTY_RESULT_PB_EXISTS_TRUE_STALE = builder.build();
243     builder.clear();
244
245     builder.setExists(false);
246     builder.setAssociatedCellCount(0);
247     EMPTY_RESULT_PB_EXISTS_FALSE =  builder.build();
248     builder.setStale(true);
249     EMPTY_RESULT_PB_EXISTS_FALSE_STALE = builder.build();
250
251     builder.clear();
252     builder.setAssociatedCellCount(0);
253     EMPTY_RESULT_PB =  builder.build();
254     builder.setStale(true);
255     EMPTY_RESULT_PB_STALE = builder.build();
256   }
257
258   /**
259    * Dynamic class loader to load filter/comparators
260    */
261   private final static ClassLoader CLASS_LOADER;
262
263   static {
264     ClassLoader parent = ProtobufUtil.class.getClassLoader();
265     Configuration conf = HBaseConfiguration.create();
266     CLASS_LOADER = new DynamicClassLoader(conf, parent);
267
268     PRIMITIVES.put(Boolean.TYPE.getName(), Boolean.TYPE);
269     PRIMITIVES.put(Byte.TYPE.getName(), Byte.TYPE);
270     PRIMITIVES.put(Character.TYPE.getName(), Character.TYPE);
271     PRIMITIVES.put(Short.TYPE.getName(), Short.TYPE);
272     PRIMITIVES.put(Integer.TYPE.getName(), Integer.TYPE);
273     PRIMITIVES.put(Long.TYPE.getName(), Long.TYPE);
274     PRIMITIVES.put(Float.TYPE.getName(), Float.TYPE);
275     PRIMITIVES.put(Double.TYPE.getName(), Double.TYPE);
276     PRIMITIVES.put(Void.TYPE.getName(), Void.TYPE);
277   }
278
279   /**
280    * Prepend the passed bytes with four bytes of magic, {@link ProtobufMagic#PB_MAGIC},
281    * to flag what follows as a protobuf in hbase.  Prepend these bytes to all content written to
282    * znodes, etc.
283    * @param bytes Bytes to decorate
284    * @return The passed <code>bytes</code> with magic prepended (Creates a new
285    * byte array that is <code>bytes.length</code> plus {@link ProtobufMagic#PB_MAGIC}.length.
286    */
287   public static byte [] prependPBMagic(final byte [] bytes) {
288     return Bytes.add(ProtobufMagic.PB_MAGIC, bytes);
289   }
290
291   /**
292    * @param bytes Bytes to check.
293    * @return True if passed <code>bytes</code> has {@link ProtobufMagic#PB_MAGIC} for a prefix.
294    */
295   public static boolean isPBMagicPrefix(final byte [] bytes) {
296     return ProtobufMagic.isPBMagicPrefix(bytes);
297   }
298
299   /**
300    * @param bytes Bytes to check.
301    * @param offset offset to start at
302    * @param len length to use
303    * @return True if passed <code>bytes</code> has {@link ProtobufMagic#PB_MAGIC} for a prefix.
304    */
305   public static boolean isPBMagicPrefix(final byte [] bytes, int offset, int len) {
306     return ProtobufMagic.isPBMagicPrefix(bytes, offset, len);
307   }
308
309   /**
310    * @param bytes bytes to check
311    * @throws DeserializationException if we are missing the pb magic prefix
312    */
313   public static void expectPBMagicPrefix(final byte [] bytes) throws DeserializationException {
314     if (!isPBMagicPrefix(bytes)) {
315       throw new DeserializationException("Missing pb magic " +
316           Bytes.toString(ProtobufMagic.PB_MAGIC) + " prefix");
317     }
318   }
319
320   /**
321    * @return Length of {@link ProtobufMagic#lengthOfPBMagic()}
322    */
323   public static int lengthOfPBMagic() {
324     return ProtobufMagic.lengthOfPBMagic();
325   }
326
327   /**
328    * Return the IOException thrown by the remote server wrapped in
329    * ServiceException as cause.
330    *
331    * @param se ServiceException that wraps IO exception thrown by the server
332    * @return Exception wrapped in ServiceException or
333    *   a new IOException that wraps the unexpected ServiceException.
334    */
335   public static IOException getRemoteException(ServiceException se) {
336     Throwable e = se.getCause();
337     if (e == null) {
338       return new IOException(se);
339     }
340     if (ExceptionUtil.isInterrupt(e)) {
341       return ExceptionUtil.asInterrupt(e);
342     }
343     if (e instanceof RemoteException) {
344       e = ((RemoteException) e).unwrapRemoteException();
345     }
346     return e instanceof IOException ? (IOException) e : new IOException(se);
347   }
348
349   /**
350    * Convert a ServerName to a protocol buffer ServerName
351    *
352    * @param serverName the ServerName to convert
353    * @return the converted protocol buffer ServerName
354    * @see #toServerName(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName)
355    */
356   public static HBaseProtos.ServerName
357       toServerName(final ServerName serverName) {
358     if (serverName == null) return null;
359     HBaseProtos.ServerName.Builder builder =
360       HBaseProtos.ServerName.newBuilder();
361     builder.setHostName(serverName.getHostname());
362     if (serverName.getPort() >= 0) {
363       builder.setPort(serverName.getPort());
364     }
365     if (serverName.getStartcode() >= 0) {
366       builder.setStartCode(serverName.getStartcode());
367     }
368     return builder.build();
369   }
370
371   /**
372    * Convert a protocol buffer ServerName to a ServerName
373    *
374    * @param proto the protocol buffer ServerName to convert
375    * @return the converted ServerName
376    */
377   public static ServerName toServerName(final HBaseProtos.ServerName proto) {
378     if (proto == null) return null;
379     String hostName = proto.getHostName();
380     long startCode = -1;
381     int port = -1;
382     if (proto.hasPort()) {
383       port = proto.getPort();
384     }
385     if (proto.hasStartCode()) {
386       startCode = proto.getStartCode();
387     }
388     return ServerName.valueOf(hostName, port, startCode);
389   }
390
391   /**
392    * Get HTableDescriptor[] from GetTableDescriptorsResponse protobuf
393    *
394    * @param proto the GetTableDescriptorsResponse
395    * @return HTableDescriptor[]
396    */
397   public static HTableDescriptor[] getHTableDescriptorArray(GetTableDescriptorsResponse proto) {
398     if (proto == null) return null;
399
400     HTableDescriptor[] ret = new HTableDescriptor[proto.getTableSchemaCount()];
401     for (int i = 0; i < proto.getTableSchemaCount(); ++i) {
402       ret[i] = convertToHTableDesc(proto.getTableSchema(i));
403     }
404     return ret;
405   }
406
407   /**
408    * get the split keys in form "byte [][]" from a CreateTableRequest proto
409    *
410    * @param proto the CreateTableRequest
411    * @return the split keys
412    */
413   public static byte [][] getSplitKeysArray(final CreateTableRequest proto) {
414     byte [][] splitKeys = new byte[proto.getSplitKeysCount()][];
415     for (int i = 0; i < proto.getSplitKeysCount(); ++i) {
416       splitKeys[i] = proto.getSplitKeys(i).toByteArray();
417     }
418     return splitKeys;
419   }
420
421   /**
422    * Convert a protobuf Durability into a client Durability
423    */
424   public static Durability toDurability(
425       final ClientProtos.MutationProto.Durability proto) {
426     switch(proto) {
427     case USE_DEFAULT:
428       return Durability.USE_DEFAULT;
429     case SKIP_WAL:
430       return Durability.SKIP_WAL;
431     case ASYNC_WAL:
432       return Durability.ASYNC_WAL;
433     case SYNC_WAL:
434       return Durability.SYNC_WAL;
435     case FSYNC_WAL:
436       return Durability.FSYNC_WAL;
437     default:
438       return Durability.USE_DEFAULT;
439     }
440   }
441
442   /**
443    * Convert a client Durability into a protbuf Durability
444    */
445   public static ClientProtos.MutationProto.Durability toDurability(
446       final Durability d) {
447     switch(d) {
448     case USE_DEFAULT:
449       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
450     case SKIP_WAL:
451       return ClientProtos.MutationProto.Durability.SKIP_WAL;
452     case ASYNC_WAL:
453       return ClientProtos.MutationProto.Durability.ASYNC_WAL;
454     case SYNC_WAL:
455       return ClientProtos.MutationProto.Durability.SYNC_WAL;
456     case FSYNC_WAL:
457       return ClientProtos.MutationProto.Durability.FSYNC_WAL;
458     default:
459       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
460     }
461   }
462
463   /**
464    * Convert a protocol buffer Get to a client Get
465    *
466    * @param proto the protocol buffer Get to convert
467    * @return the converted client Get
468    * @throws IOException
469    */
470   public static Get toGet(
471       final ClientProtos.Get proto) throws IOException {
472     if (proto == null) return null;
473     byte[] row = proto.getRow().toByteArray();
474     Get get = new Get(row);
475     if (proto.hasCacheBlocks()) {
476       get.setCacheBlocks(proto.getCacheBlocks());
477     }
478     if (proto.hasMaxVersions()) {
479       get.setMaxVersions(proto.getMaxVersions());
480     }
481     if (proto.hasStoreLimit()) {
482       get.setMaxResultsPerColumnFamily(proto.getStoreLimit());
483     }
484     if (proto.hasStoreOffset()) {
485       get.setRowOffsetPerColumnFamily(proto.getStoreOffset());
486     }
487     if (proto.getCfTimeRangeCount() > 0) {
488       for (HBaseProtos.ColumnFamilyTimeRange cftr : proto.getCfTimeRangeList()) {
489         TimeRange timeRange = protoToTimeRange(cftr.getTimeRange());
490         get.setColumnFamilyTimeRange(cftr.getColumnFamily().toByteArray(),
491             timeRange.getMin(), timeRange.getMax());
492       }
493     }
494     if (proto.hasTimeRange()) {
495       TimeRange timeRange = protoToTimeRange(proto.getTimeRange());
496       get.setTimeRange(timeRange.getMin(), timeRange.getMax());
497     }
498     if (proto.hasFilter()) {
499       FilterProtos.Filter filter = proto.getFilter();
500       get.setFilter(ProtobufUtil.toFilter(filter));
501     }
502     for (NameBytesPair attribute: proto.getAttributeList()) {
503       get.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
504     }
505     if (proto.getColumnCount() > 0) {
506       for (Column column: proto.getColumnList()) {
507         byte[] family = column.getFamily().toByteArray();
508         if (column.getQualifierCount() > 0) {
509           for (ByteString qualifier: column.getQualifierList()) {
510             get.addColumn(family, qualifier.toByteArray());
511           }
512         } else {
513           get.addFamily(family);
514         }
515       }
516     }
517     if (proto.hasExistenceOnly() && proto.getExistenceOnly()){
518       get.setCheckExistenceOnly(true);
519     }
520     if (proto.hasConsistency()) {
521       get.setConsistency(toConsistency(proto.getConsistency()));
522     }
523     return get;
524   }
525
526   public static Consistency toConsistency(ClientProtos.Consistency consistency) {
527     switch (consistency) {
528       case STRONG : return Consistency.STRONG;
529       case TIMELINE : return Consistency.TIMELINE;
530       default : return Consistency.STRONG;
531     }
532   }
533
534   public static ClientProtos.Consistency toConsistency(Consistency consistency) {
535     switch (consistency) {
536       case STRONG : return ClientProtos.Consistency.STRONG;
537       case TIMELINE : return ClientProtos.Consistency.TIMELINE;
538       default : return ClientProtos.Consistency.STRONG;
539     }
540   }
541
542   /**
543    * Convert a protocol buffer Mutate to a Put.
544    *
545    * @param proto The protocol buffer MutationProto to convert
546    * @return A client Put.
547    * @throws IOException
548    */
549   public static Put toPut(final MutationProto proto)
550   throws IOException {
551     return toPut(proto, null);
552   }
553
554   /**
555    * Convert a protocol buffer Mutate to a Put.
556    *
557    * @param proto The protocol buffer MutationProto to convert
558    * @param cellScanner If non-null, the Cell data that goes with this proto.
559    * @return A client Put.
560    * @throws IOException
561    */
562   public static Put toPut(final MutationProto proto, final CellScanner cellScanner)
563   throws IOException {
564     // TODO: Server-side at least why do we convert back to the Client types?  Why not just pb it?
565     MutationType type = proto.getMutateType();
566     assert type == MutationType.PUT: type.name();
567     long timestamp = proto.hasTimestamp()? proto.getTimestamp(): HConstants.LATEST_TIMESTAMP;
568     Put put = proto.hasRow() ? new Put(proto.getRow().toByteArray(), timestamp) : null;
569     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
570     if (cellCount > 0) {
571       // The proto has metadata only and the data is separate to be found in the cellScanner.
572       if (cellScanner == null) {
573         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
574             toShortString(proto));
575       }
576       for (int i = 0; i < cellCount; i++) {
577         if (!cellScanner.advance()) {
578           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
579             " no cell returned: " + toShortString(proto));
580         }
581         Cell cell = cellScanner.current();
582         if (put == null) {
583           put = new Put(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
584         }
585         put.add(cell);
586       }
587     } else {
588       if (put == null) {
589         throw new IllegalArgumentException("row cannot be null");
590       }
591       // The proto has the metadata and the data itself
592       for (ColumnValue column: proto.getColumnValueList()) {
593         byte[] family = column.getFamily().toByteArray();
594         for (QualifierValue qv: column.getQualifierValueList()) {
595           if (!qv.hasValue()) {
596             throw new DoNotRetryIOException(
597                 "Missing required field: qualifier value");
598           }
599           ByteBuffer qualifier =
600               qv.hasQualifier() ? qv.getQualifier().asReadOnlyByteBuffer() : null;
601           ByteBuffer value =
602               qv.hasValue() ? qv.getValue().asReadOnlyByteBuffer() : null;
603           long ts = timestamp;
604           if (qv.hasTimestamp()) {
605             ts = qv.getTimestamp();
606           }
607           byte[] allTagsBytes;
608           if (qv.hasTags()) {
609             allTagsBytes = qv.getTags().toByteArray();
610             if(qv.hasDeleteType()) {
611               byte[] qual = qv.hasQualifier() ? qv.getQualifier().toByteArray() : null;
612               put.add(new KeyValue(proto.getRow().toByteArray(), family, qual, ts,
613                   fromDeleteType(qv.getDeleteType()), null, allTagsBytes));
614             } else {
615               List<Tag> tags = TagUtil.asList(allTagsBytes, 0, (short)allTagsBytes.length);
616               Tag[] tagsArray = new Tag[tags.size()];
617               put.addImmutable(family, qualifier, ts, value, tags.toArray(tagsArray));
618             }
619           } else {
620             if(qv.hasDeleteType()) {
621               byte[] qual = qv.hasQualifier() ? qv.getQualifier().toByteArray() : null;
622               put.add(new KeyValue(proto.getRow().toByteArray(), family, qual, ts,
623                   fromDeleteType(qv.getDeleteType())));
624             } else{
625               put.addImmutable(family, qualifier, ts, value);
626             }
627           }
628         }
629       }
630     }
631     put.setDurability(toDurability(proto.getDurability()));
632     for (NameBytesPair attribute: proto.getAttributeList()) {
633       put.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
634     }
635     return put;
636   }
637
638   /**
639    * Convert a protocol buffer Mutate to a Delete
640    *
641    * @param proto the protocol buffer Mutate to convert
642    * @return the converted client Delete
643    * @throws IOException
644    */
645   public static Delete toDelete(final MutationProto proto)
646   throws IOException {
647     return toDelete(proto, null);
648   }
649
650   /**
651    * Convert a protocol buffer Mutate to a Delete
652    *
653    * @param proto the protocol buffer Mutate to convert
654    * @param cellScanner if non-null, the data that goes with this delete.
655    * @return the converted client Delete
656    * @throws IOException
657    */
658   public static Delete toDelete(final MutationProto proto, final CellScanner cellScanner)
659   throws IOException {
660     MutationType type = proto.getMutateType();
661     assert type == MutationType.DELETE : type.name();
662     long timestamp = proto.hasTimestamp() ? proto.getTimestamp() : HConstants.LATEST_TIMESTAMP;
663     Delete delete = proto.hasRow() ? new Delete(proto.getRow().toByteArray(), timestamp) : null;
664     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
665     if (cellCount > 0) {
666       // The proto has metadata only and the data is separate to be found in the cellScanner.
667       if (cellScanner == null) {
668         // TextFormat should be fine for a Delete since it carries no data, just coordinates.
669         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
670           TextFormat.shortDebugString(proto));
671       }
672       for (int i = 0; i < cellCount; i++) {
673         if (!cellScanner.advance()) {
674           // TextFormat should be fine for a Delete since it carries no data, just coordinates.
675           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
676             " no cell returned: " + TextFormat.shortDebugString(proto));
677         }
678         Cell cell = cellScanner.current();
679         if (delete == null) {
680           delete =
681             new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
682         }
683         delete.addDeleteMarker(cell);
684       }
685     } else {
686       if (delete == null) {
687         throw new IllegalArgumentException("row cannot be null");
688       }
689       for (ColumnValue column: proto.getColumnValueList()) {
690         byte[] family = column.getFamily().toByteArray();
691         for (QualifierValue qv: column.getQualifierValueList()) {
692           DeleteType deleteType = qv.getDeleteType();
693           byte[] qualifier = null;
694           if (qv.hasQualifier()) {
695             qualifier = qv.getQualifier().toByteArray();
696           }
697           long ts = HConstants.LATEST_TIMESTAMP;
698           if (qv.hasTimestamp()) {
699             ts = qv.getTimestamp();
700           }
701           if (deleteType == DeleteType.DELETE_ONE_VERSION) {
702             delete.addColumn(family, qualifier, ts);
703           } else if (deleteType == DeleteType.DELETE_MULTIPLE_VERSIONS) {
704             delete.addColumns(family, qualifier, ts);
705           } else if (deleteType == DeleteType.DELETE_FAMILY_VERSION) {
706             delete.addFamilyVersion(family, ts);
707           } else {
708             delete.addFamily(family, ts);
709           }
710         }
711       }
712     }
713     delete.setDurability(toDurability(proto.getDurability()));
714     for (NameBytesPair attribute: proto.getAttributeList()) {
715       delete.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
716     }
717     return delete;
718   }
719
720   /**
721    * Convert a protocol buffer Mutate to an Append
722    * @param cellScanner
723    * @param proto the protocol buffer Mutate to convert
724    * @return the converted client Append
725    * @throws IOException
726    */
727   public static Append toAppend(final MutationProto proto, final CellScanner cellScanner)
728   throws IOException {
729     MutationType type = proto.getMutateType();
730     assert type == MutationType.APPEND : type.name();
731     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
732     Append append = null;
733     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
734     if (cellCount > 0) {
735       // The proto has metadata only and the data is separate to be found in the cellScanner.
736       if (cellScanner == null) {
737         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
738           toShortString(proto));
739       }
740       for (int i = 0; i < cellCount; i++) {
741         if (!cellScanner.advance()) {
742           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
743             " no cell returned: " + toShortString(proto));
744         }
745         Cell cell = cellScanner.current();
746         if (append == null) {
747           append = new Append(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
748         }
749         append.add(cell);
750       }
751     } else {
752       append = new Append(row);
753       for (ColumnValue column: proto.getColumnValueList()) {
754         byte[] family = column.getFamily().toByteArray();
755         for (QualifierValue qv: column.getQualifierValueList()) {
756           byte[] qualifier = qv.getQualifier().toByteArray();
757           if (!qv.hasValue()) {
758             throw new DoNotRetryIOException(
759               "Missing required field: qualifier value");
760           }
761           byte[] value = qv.getValue().toByteArray();
762           byte[] tags = null;
763           if (qv.hasTags()) {
764             tags = qv.getTags().toByteArray();
765           }
766           append.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
767               KeyValue.Type.Put, value, tags));
768         }
769       }
770     }
771     append.setDurability(toDurability(proto.getDurability()));
772     for (NameBytesPair attribute: proto.getAttributeList()) {
773       append.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
774     }
775     return append;
776   }
777
778   /**
779    * Convert a MutateRequest to Mutation
780    *
781    * @param proto the protocol buffer Mutate to convert
782    * @return the converted Mutation
783    * @throws IOException
784    */
785   public static Mutation toMutation(final MutationProto proto) throws IOException {
786     MutationType type = proto.getMutateType();
787     if (type == MutationType.APPEND) {
788       return toAppend(proto, null);
789     }
790     if (type == MutationType.DELETE) {
791       return toDelete(proto, null);
792     }
793     if (type == MutationType.PUT) {
794       return toPut(proto, null);
795     }
796     throw new IOException("Unknown mutation type " + type);
797   }
798
799   /**
800    * Convert a protocol buffer Mutate to an Increment
801    *
802    * @param proto the protocol buffer Mutate to convert
803    * @return the converted client Increment
804    * @throws IOException
805    */
806   public static Increment toIncrement(final MutationProto proto, final CellScanner cellScanner)
807   throws IOException {
808     MutationType type = proto.getMutateType();
809     assert type == MutationType.INCREMENT : type.name();
810     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
811     Increment increment = null;
812     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
813     if (cellCount > 0) {
814       // The proto has metadata only and the data is separate to be found in the cellScanner.
815       if (cellScanner == null) {
816         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
817           TextFormat.shortDebugString(proto));
818       }
819       for (int i = 0; i < cellCount; i++) {
820         if (!cellScanner.advance()) {
821           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
822             " no cell returned: " + TextFormat.shortDebugString(proto));
823         }
824         Cell cell = cellScanner.current();
825         if (increment == null) {
826           increment = new Increment(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
827         }
828         increment.add(cell);
829       }
830     } else {
831       increment = new Increment(row);
832       for (ColumnValue column: proto.getColumnValueList()) {
833         byte[] family = column.getFamily().toByteArray();
834         for (QualifierValue qv: column.getQualifierValueList()) {
835           byte[] qualifier = qv.getQualifier().toByteArray();
836           if (!qv.hasValue()) {
837             throw new DoNotRetryIOException("Missing required field: qualifier value");
838           }
839           byte[] value = qv.getValue().toByteArray();
840           byte[] tags = null;
841           if (qv.hasTags()) {
842             tags = qv.getTags().toByteArray();
843           }
844           increment.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
845               KeyValue.Type.Put, value, tags));
846         }
847       }
848     }
849     if (proto.hasTimeRange()) {
850       TimeRange timeRange = protoToTimeRange(proto.getTimeRange());
851       increment.setTimeRange(timeRange.getMin(), timeRange.getMax());
852     }
853     increment.setDurability(toDurability(proto.getDurability()));
854     for (NameBytesPair attribute : proto.getAttributeList()) {
855       increment.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
856     }
857     return increment;
858   }
859
860   /**
861    * Convert a client Scan to a protocol buffer Scan
862    *
863    * @param scan the client Scan to convert
864    * @return the converted protocol buffer Scan
865    * @throws IOException
866    */
867   public static ClientProtos.Scan toScan(
868       final Scan scan) throws IOException {
869     ClientProtos.Scan.Builder scanBuilder =
870       ClientProtos.Scan.newBuilder();
871     scanBuilder.setCacheBlocks(scan.getCacheBlocks());
872     if (scan.getBatch() > 0) {
873       scanBuilder.setBatchSize(scan.getBatch());
874     }
875     if (scan.getMaxResultSize() > 0) {
876       scanBuilder.setMaxResultSize(scan.getMaxResultSize());
877     }
878     if (scan.isSmall()) {
879       scanBuilder.setSmall(scan.isSmall());
880     }
881     if (scan.getAllowPartialResults()) {
882       scanBuilder.setAllowPartialResults(scan.getAllowPartialResults());
883     }
884     Boolean loadColumnFamiliesOnDemand = scan.getLoadColumnFamiliesOnDemandValue();
885     if (loadColumnFamiliesOnDemand != null) {
886       scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand.booleanValue());
887     }
888     scanBuilder.setMaxVersions(scan.getMaxVersions());
889     for (Entry<byte[], TimeRange> cftr : scan.getColumnFamilyTimeRange().entrySet()) {
890       HBaseProtos.ColumnFamilyTimeRange.Builder b = HBaseProtos.ColumnFamilyTimeRange.newBuilder();
891       b.setColumnFamily(ByteString.copyFrom(cftr.getKey()));
892       b.setTimeRange(timeRangeToProto(cftr.getValue()));
893       scanBuilder.addCfTimeRange(b);
894     }
895     TimeRange timeRange = scan.getTimeRange();
896     if (!timeRange.isAllTime()) {
897       HBaseProtos.TimeRange.Builder timeRangeBuilder =
898         HBaseProtos.TimeRange.newBuilder();
899       timeRangeBuilder.setFrom(timeRange.getMin());
900       timeRangeBuilder.setTo(timeRange.getMax());
901       scanBuilder.setTimeRange(timeRangeBuilder.build());
902     }
903     Map<String, byte[]> attributes = scan.getAttributesMap();
904     if (!attributes.isEmpty()) {
905       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
906       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
907         attributeBuilder.setName(attribute.getKey());
908         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
909         scanBuilder.addAttribute(attributeBuilder.build());
910       }
911     }
912     byte[] startRow = scan.getStartRow();
913     if (startRow != null && startRow.length > 0) {
914       scanBuilder.setStartRow(ByteStringer.wrap(startRow));
915     }
916     byte[] stopRow = scan.getStopRow();
917     if (stopRow != null && stopRow.length > 0) {
918       scanBuilder.setStopRow(ByteStringer.wrap(stopRow));
919     }
920     if (scan.hasFilter()) {
921       scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
922     }
923     if (scan.hasFamilies()) {
924       Column.Builder columnBuilder = Column.newBuilder();
925       for (Map.Entry<byte[],NavigableSet<byte []>>
926           family: scan.getFamilyMap().entrySet()) {
927         columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
928         NavigableSet<byte []> qualifiers = family.getValue();
929         columnBuilder.clearQualifier();
930         if (qualifiers != null && qualifiers.size() > 0) {
931           for (byte [] qualifier: qualifiers) {
932             columnBuilder.addQualifier(ByteStringer.wrap(qualifier));
933           }
934         }
935         scanBuilder.addColumn(columnBuilder.build());
936       }
937     }
938     if (scan.getMaxResultsPerColumnFamily() >= 0) {
939       scanBuilder.setStoreLimit(scan.getMaxResultsPerColumnFamily());
940     }
941     if (scan.getRowOffsetPerColumnFamily() > 0) {
942       scanBuilder.setStoreOffset(scan.getRowOffsetPerColumnFamily());
943     }
944     if (scan.isReversed()) {
945       scanBuilder.setReversed(scan.isReversed());
946     }
947     if (scan.getConsistency() == Consistency.TIMELINE) {
948       scanBuilder.setConsistency(toConsistency(scan.getConsistency()));
949     }
950     if (scan.getCaching() > 0) {
951       scanBuilder.setCaching(scan.getCaching());
952     }
953     return scanBuilder.build();
954   }
955
956   /**
957    * Convert a protocol buffer Scan to a client Scan
958    *
959    * @param proto the protocol buffer Scan to convert
960    * @return the converted client Scan
961    * @throws IOException
962    */
963   public static Scan toScan(
964       final ClientProtos.Scan proto) throws IOException {
965     byte [] startRow = HConstants.EMPTY_START_ROW;
966     byte [] stopRow  = HConstants.EMPTY_END_ROW;
967     if (proto.hasStartRow()) {
968       startRow = proto.getStartRow().toByteArray();
969     }
970     if (proto.hasStopRow()) {
971       stopRow = proto.getStopRow().toByteArray();
972     }
973     Scan scan = new Scan(startRow, stopRow);
974     if (proto.hasCacheBlocks()) {
975       scan.setCacheBlocks(proto.getCacheBlocks());
976     }
977     if (proto.hasMaxVersions()) {
978       scan.setMaxVersions(proto.getMaxVersions());
979     }
980     if (proto.hasStoreLimit()) {
981       scan.setMaxResultsPerColumnFamily(proto.getStoreLimit());
982     }
983     if (proto.hasStoreOffset()) {
984       scan.setRowOffsetPerColumnFamily(proto.getStoreOffset());
985     }
986     if (proto.hasLoadColumnFamiliesOnDemand()) {
987       scan.setLoadColumnFamiliesOnDemand(proto.getLoadColumnFamiliesOnDemand());
988     }
989     if (proto.getCfTimeRangeCount() > 0) {
990       for (HBaseProtos.ColumnFamilyTimeRange cftr : proto.getCfTimeRangeList()) {
991         TimeRange timeRange = protoToTimeRange(cftr.getTimeRange());
992         scan.setColumnFamilyTimeRange(cftr.getColumnFamily().toByteArray(),
993             timeRange.getMin(), timeRange.getMax());
994       }
995     }
996     if (proto.hasTimeRange()) {
997       TimeRange timeRange = protoToTimeRange(proto.getTimeRange());
998       scan.setTimeRange(timeRange.getMin(), timeRange.getMax());
999     }
1000     if (proto.hasFilter()) {
1001       FilterProtos.Filter filter = proto.getFilter();
1002       scan.setFilter(ProtobufUtil.toFilter(filter));
1003     }
1004     if (proto.hasBatchSize()) {
1005       scan.setBatch(proto.getBatchSize());
1006     }
1007     if (proto.hasMaxResultSize()) {
1008       scan.setMaxResultSize(proto.getMaxResultSize());
1009     }
1010     if (proto.hasSmall()) {
1011       scan.setSmall(proto.getSmall());
1012     }
1013     if (proto.hasAllowPartialResults()) {
1014       scan.setAllowPartialResults(proto.getAllowPartialResults());
1015     }
1016     for (NameBytesPair attribute: proto.getAttributeList()) {
1017       scan.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
1018     }
1019     if (proto.getColumnCount() > 0) {
1020       for (Column column: proto.getColumnList()) {
1021         byte[] family = column.getFamily().toByteArray();
1022         if (column.getQualifierCount() > 0) {
1023           for (ByteString qualifier: column.getQualifierList()) {
1024             scan.addColumn(family, qualifier.toByteArray());
1025           }
1026         } else {
1027           scan.addFamily(family);
1028         }
1029       }
1030     }
1031     if (proto.hasReversed()) {
1032       scan.setReversed(proto.getReversed());
1033     }
1034     if (proto.hasConsistency()) {
1035       scan.setConsistency(toConsistency(proto.getConsistency()));
1036     }
1037     if (proto.hasCaching()) {
1038       scan.setCaching(proto.getCaching());
1039     }
1040     return scan;
1041   }
1042
1043   /**
1044    * Create a protocol buffer Get based on a client Get.
1045    *
1046    * @param get the client Get
1047    * @return a protocol buffer Get
1048    * @throws IOException
1049    */
1050   public static ClientProtos.Get toGet(
1051       final Get get) throws IOException {
1052     ClientProtos.Get.Builder builder =
1053       ClientProtos.Get.newBuilder();
1054     builder.setRow(ByteStringer.wrap(get.getRow()));
1055     builder.setCacheBlocks(get.getCacheBlocks());
1056     builder.setMaxVersions(get.getMaxVersions());
1057     if (get.getFilter() != null) {
1058       builder.setFilter(ProtobufUtil.toFilter(get.getFilter()));
1059     }
1060     for (Entry<byte[], TimeRange> cftr : get.getColumnFamilyTimeRange().entrySet()) {
1061       HBaseProtos.ColumnFamilyTimeRange.Builder b = HBaseProtos.ColumnFamilyTimeRange.newBuilder();
1062       b.setColumnFamily(ByteString.copyFrom(cftr.getKey()));
1063       b.setTimeRange(timeRangeToProto(cftr.getValue()));
1064       builder.addCfTimeRange(b);
1065     }
1066     TimeRange timeRange = get.getTimeRange();
1067     if (!timeRange.isAllTime()) {
1068       HBaseProtos.TimeRange.Builder timeRangeBuilder =
1069         HBaseProtos.TimeRange.newBuilder();
1070       timeRangeBuilder.setFrom(timeRange.getMin());
1071       timeRangeBuilder.setTo(timeRange.getMax());
1072       builder.setTimeRange(timeRangeBuilder.build());
1073     }
1074     Map<String, byte[]> attributes = get.getAttributesMap();
1075     if (!attributes.isEmpty()) {
1076       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1077       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
1078         attributeBuilder.setName(attribute.getKey());
1079         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1080         builder.addAttribute(attributeBuilder.build());
1081       }
1082     }
1083     if (get.hasFamilies()) {
1084       Column.Builder columnBuilder = Column.newBuilder();
1085       Map<byte[], NavigableSet<byte[]>> families = get.getFamilyMap();
1086       for (Map.Entry<byte[], NavigableSet<byte[]>> family: families.entrySet()) {
1087         NavigableSet<byte[]> qualifiers = family.getValue();
1088         columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1089         columnBuilder.clearQualifier();
1090         if (qualifiers != null && qualifiers.size() > 0) {
1091           for (byte[] qualifier: qualifiers) {
1092             columnBuilder.addQualifier(ByteStringer.wrap(qualifier));
1093           }
1094         }
1095         builder.addColumn(columnBuilder.build());
1096       }
1097     }
1098     if (get.getMaxResultsPerColumnFamily() >= 0) {
1099       builder.setStoreLimit(get.getMaxResultsPerColumnFamily());
1100     }
1101     if (get.getRowOffsetPerColumnFamily() > 0) {
1102       builder.setStoreOffset(get.getRowOffsetPerColumnFamily());
1103     }
1104     if (get.isCheckExistenceOnly()){
1105       builder.setExistenceOnly(true);
1106     }
1107     if (get.getConsistency() != null && get.getConsistency() != Consistency.STRONG) {
1108       builder.setConsistency(toConsistency(get.getConsistency()));
1109     }
1110
1111     return builder.build();
1112   }
1113
1114   static void setTimeRange(final MutationProto.Builder builder, final TimeRange timeRange) {
1115     if (!timeRange.isAllTime()) {
1116       HBaseProtos.TimeRange.Builder timeRangeBuilder =
1117         HBaseProtos.TimeRange.newBuilder();
1118       timeRangeBuilder.setFrom(timeRange.getMin());
1119       timeRangeBuilder.setTo(timeRange.getMax());
1120       builder.setTimeRange(timeRangeBuilder.build());
1121     }
1122   }
1123
1124   /**
1125    * Convert a client Increment to a protobuf Mutate.
1126    *
1127    * @param increment
1128    * @return the converted mutate
1129    */
1130   public static MutationProto toMutation(
1131     final Increment increment, final MutationProto.Builder builder, long nonce) {
1132     builder.setRow(ByteStringer.wrap(increment.getRow()));
1133     builder.setMutateType(MutationType.INCREMENT);
1134     builder.setDurability(toDurability(increment.getDurability()));
1135     if (nonce != HConstants.NO_NONCE) {
1136       builder.setNonce(nonce);
1137     }
1138     TimeRange timeRange = increment.getTimeRange();
1139     setTimeRange(builder, timeRange);
1140     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1141     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1142     for (Map.Entry<byte[], List<Cell>> family: increment.getFamilyCellMap().entrySet()) {
1143       columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1144       columnBuilder.clearQualifierValue();
1145       List<Cell> values = family.getValue();
1146       if (values != null && values.size() > 0) {
1147         for (Cell cell: values) {
1148           valueBuilder.clear();
1149           valueBuilder.setQualifier(ByteStringer.wrap(
1150               cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
1151           valueBuilder.setValue(ByteStringer.wrap(
1152               cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
1153           if (cell.getTagsLength() > 0) {
1154             valueBuilder.setTags(ByteStringer.wrap(cell.getTagsArray(),
1155                 cell.getTagsOffset(), cell.getTagsLength()));
1156           }
1157           columnBuilder.addQualifierValue(valueBuilder.build());
1158         }
1159       }
1160       builder.addColumnValue(columnBuilder.build());
1161     }
1162     Map<String, byte[]> attributes = increment.getAttributesMap();
1163     if (!attributes.isEmpty()) {
1164       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1165       for (Map.Entry<String, byte[]> attribute : attributes.entrySet()) {
1166         attributeBuilder.setName(attribute.getKey());
1167         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1168         builder.addAttribute(attributeBuilder.build());
1169       }
1170     }
1171     return builder.build();
1172   }
1173
1174   public static MutationProto toMutation(final MutationType type, final Mutation mutation)
1175     throws IOException {
1176     return toMutation(type, mutation, HConstants.NO_NONCE);
1177   }
1178
1179   /**
1180    * Create a protocol buffer Mutate based on a client Mutation
1181    *
1182    * @param type
1183    * @param mutation
1184    * @return a protobuf'd Mutation
1185    * @throws IOException
1186    */
1187   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1188     final long nonce) throws IOException {
1189     return toMutation(type, mutation, MutationProto.newBuilder(), nonce);
1190   }
1191
1192   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1193       MutationProto.Builder builder) throws IOException {
1194     return toMutation(type, mutation, builder, HConstants.NO_NONCE);
1195   }
1196
1197   @SuppressWarnings("deprecation")
1198   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1199       MutationProto.Builder builder, long nonce)
1200   throws IOException {
1201     builder = getMutationBuilderAndSetCommonFields(type, mutation, builder);
1202     if (nonce != HConstants.NO_NONCE) {
1203       builder.setNonce(nonce);
1204     }
1205     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1206     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1207     for (Map.Entry<byte[],List<Cell>> family: mutation.getFamilyCellMap().entrySet()) {
1208       columnBuilder.clear();
1209       columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1210       for (Cell cell: family.getValue()) {
1211         valueBuilder.clear();
1212         valueBuilder.setQualifier(ByteStringer.wrap(
1213             cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
1214         valueBuilder.setValue(ByteStringer.wrap(
1215             cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
1216         valueBuilder.setTimestamp(cell.getTimestamp());
1217         if (type == MutationType.DELETE || (type == MutationType.PUT && CellUtil.isDelete(cell))) {
1218           KeyValue.Type keyValueType = KeyValue.Type.codeToType(cell.getTypeByte());
1219           valueBuilder.setDeleteType(toDeleteType(keyValueType));
1220         }
1221         columnBuilder.addQualifierValue(valueBuilder.build());
1222       }
1223       builder.addColumnValue(columnBuilder.build());
1224     }
1225     return builder.build();
1226   }
1227
1228   /**
1229    * Create a protocol buffer MutationProto based on a client Mutation. Does NOT include data.
1230    * Understanding is that the Cell will be transported other than via protobuf.
1231    * @param type
1232    * @param mutation
1233    * @param builder
1234    * @return a protobuf'd Mutation
1235    * @throws IOException
1236    */
1237   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1238       final MutationProto.Builder builder)  throws IOException {
1239     return toMutationNoData(type, mutation, builder, HConstants.NO_NONCE);
1240   }
1241
1242   /**
1243    * Create a protocol buffer MutationProto based on a client Mutation.  Does NOT include data.
1244    * Understanding is that the Cell will be transported other than via protobuf.
1245    * @param type
1246    * @param mutation
1247    * @return a protobuf'd Mutation
1248    * @throws IOException
1249    */
1250   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation)
1251   throws IOException {
1252     MutationProto.Builder builder =  MutationProto.newBuilder();
1253     return toMutationNoData(type, mutation, builder);
1254   }
1255
1256   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1257       final MutationProto.Builder builder, long nonce) throws IOException {
1258     getMutationBuilderAndSetCommonFields(type, mutation, builder);
1259     builder.setAssociatedCellCount(mutation.size());
1260     if (mutation instanceof Increment) {
1261       setTimeRange(builder, ((Increment)mutation).getTimeRange());
1262     }
1263     if (nonce != HConstants.NO_NONCE) {
1264       builder.setNonce(nonce);
1265     }
1266     return builder.build();
1267   }
1268
1269   /**
1270    * Code shared by {@link #toMutation(MutationType, Mutation)} and
1271    * {@link #toMutationNoData(MutationType, Mutation)}
1272    * @param type
1273    * @param mutation
1274    * @return A partly-filled out protobuf'd Mutation.
1275    */
1276   private static MutationProto.Builder getMutationBuilderAndSetCommonFields(final MutationType type,
1277       final Mutation mutation, MutationProto.Builder builder) {
1278     builder.setRow(ByteStringer.wrap(mutation.getRow()));
1279     builder.setMutateType(type);
1280     builder.setDurability(toDurability(mutation.getDurability()));
1281     builder.setTimestamp(mutation.getTimeStamp());
1282     Map<String, byte[]> attributes = mutation.getAttributesMap();
1283     if (!attributes.isEmpty()) {
1284       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1285       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
1286         attributeBuilder.setName(attribute.getKey());
1287         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1288         builder.addAttribute(attributeBuilder.build());
1289       }
1290     }
1291     return builder;
1292   }
1293
1294   /**
1295    * Convert a client Result to a protocol buffer Result
1296    *
1297    * @param result the client Result to convert
1298    * @return the converted protocol buffer Result
1299    */
1300   public static ClientProtos.Result toResult(final Result result) {
1301     if (result.getExists() != null) {
1302       return toResult(result.getExists(), result.isStale());
1303     }
1304
1305     Cell[] cells = result.rawCells();
1306     if (cells == null || cells.length == 0) {
1307       return result.isStale() ? EMPTY_RESULT_PB_STALE : EMPTY_RESULT_PB;
1308     }
1309
1310     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1311     for (Cell c : cells) {
1312       builder.addCell(toCell(c));
1313     }
1314
1315     builder.setStale(result.isStale());
1316     builder.setPartial(result.isPartial());
1317
1318     return builder.build();
1319   }
1320
1321   /**
1322    * Convert a client Result to a protocol buffer Result
1323    *
1324    * @param existence the client existence to send
1325    * @return the converted protocol buffer Result
1326    */
1327   public static ClientProtos.Result toResult(final boolean existence, boolean stale) {
1328     if (stale){
1329       return existence ? EMPTY_RESULT_PB_EXISTS_TRUE_STALE : EMPTY_RESULT_PB_EXISTS_FALSE_STALE;
1330     } else {
1331       return existence ? EMPTY_RESULT_PB_EXISTS_TRUE : EMPTY_RESULT_PB_EXISTS_FALSE;
1332     }
1333   }
1334
1335   /**
1336    * Convert a client Result to a protocol buffer Result.
1337    * The pb Result does not include the Cell data.  That is for transport otherwise.
1338    *
1339    * @param result the client Result to convert
1340    * @return the converted protocol buffer Result
1341    */
1342   public static ClientProtos.Result toResultNoData(final Result result) {
1343     if (result.getExists() != null) return toResult(result.getExists(), result.isStale());
1344     int size = result.size();
1345     if (size == 0) return result.isStale() ? EMPTY_RESULT_PB_STALE : EMPTY_RESULT_PB;
1346     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1347     builder.setAssociatedCellCount(size);
1348     builder.setStale(result.isStale());
1349     return builder.build();
1350   }
1351
1352   /**
1353    * Convert a protocol buffer Result to a client Result
1354    *
1355    * @param proto the protocol buffer Result to convert
1356    * @return the converted client Result
1357    */
1358   public static Result toResult(final ClientProtos.Result proto) {
1359     if (proto.hasExists()) {
1360       if (proto.getStale()) {
1361         return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE_STALE :EMPTY_RESULT_EXISTS_FALSE_STALE;
1362       }
1363       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1364     }
1365
1366     List<CellProtos.Cell> values = proto.getCellList();
1367     if (values.isEmpty()){
1368       return proto.getStale() ? EMPTY_RESULT_STALE : EMPTY_RESULT;
1369     }
1370
1371     List<Cell> cells = new ArrayList<Cell>(values.size());
1372     for (CellProtos.Cell c : values) {
1373       cells.add(toCell(c));
1374     }
1375     return Result.create(cells, null, proto.getStale(), proto.getPartial());
1376   }
1377
1378   /**
1379    * Convert a protocol buffer Result to a client Result
1380    *
1381    * @param proto the protocol buffer Result to convert
1382    * @param scanner Optional cell scanner.
1383    * @return the converted client Result
1384    * @throws IOException
1385    */
1386   public static Result toResult(final ClientProtos.Result proto, final CellScanner scanner)
1387   throws IOException {
1388     List<CellProtos.Cell> values = proto.getCellList();
1389
1390     if (proto.hasExists()) {
1391       if ((values != null && !values.isEmpty()) ||
1392           (proto.hasAssociatedCellCount() && proto.getAssociatedCellCount() > 0)) {
1393         throw new IllegalArgumentException("bad proto: exists with cells is no allowed " + proto);
1394       }
1395       if (proto.getStale()) {
1396         return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE_STALE :EMPTY_RESULT_EXISTS_FALSE_STALE;
1397       }
1398       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1399     }
1400
1401     // TODO: Unit test that has some Cells in scanner and some in the proto.
1402     List<Cell> cells = null;
1403     if (proto.hasAssociatedCellCount()) {
1404       int count = proto.getAssociatedCellCount();
1405       cells = new ArrayList<Cell>(count + values.size());
1406       for (int i = 0; i < count; i++) {
1407         if (!scanner.advance()) throw new IOException("Failed get " + i + " of " + count);
1408         cells.add(scanner.current());
1409       }
1410     }
1411
1412     if (!values.isEmpty()){
1413       if (cells == null) cells = new ArrayList<Cell>(values.size());
1414       for (CellProtos.Cell c: values) {
1415         cells.add(toCell(c));
1416       }
1417     }
1418
1419     return (cells == null || cells.isEmpty())
1420         ? (proto.getStale() ? EMPTY_RESULT_STALE : EMPTY_RESULT)
1421         : Result.create(cells, null, proto.getStale());
1422   }
1423
1424
1425   /**
1426    * Convert a ByteArrayComparable to a protocol buffer Comparator
1427    *
1428    * @param comparator the ByteArrayComparable to convert
1429    * @return the converted protocol buffer Comparator
1430    */
1431   public static ComparatorProtos.Comparator toComparator(ByteArrayComparable comparator) {
1432     ComparatorProtos.Comparator.Builder builder = ComparatorProtos.Comparator.newBuilder();
1433     builder.setName(comparator.getClass().getName());
1434     builder.setSerializedComparator(ByteStringer.wrap(comparator.toByteArray()));
1435     return builder.build();
1436   }
1437
1438   /**
1439    * Convert a protocol buffer Comparator to a ByteArrayComparable
1440    *
1441    * @param proto the protocol buffer Comparator to convert
1442    * @return the converted ByteArrayComparable
1443    */
1444   @SuppressWarnings("unchecked")
1445   public static ByteArrayComparable toComparator(ComparatorProtos.Comparator proto)
1446   throws IOException {
1447     String type = proto.getName();
1448     String funcName = "parseFrom";
1449     byte [] value = proto.getSerializedComparator().toByteArray();
1450     try {
1451       Class<? extends ByteArrayComparable> c =
1452         (Class<? extends ByteArrayComparable>)Class.forName(type, true, CLASS_LOADER);
1453       Method parseFrom = c.getMethod(funcName, byte[].class);
1454       if (parseFrom == null) {
1455         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1456       }
1457       return (ByteArrayComparable)parseFrom.invoke(null, value);
1458     } catch (Exception e) {
1459       throw new IOException(e);
1460     }
1461   }
1462
1463   /**
1464    * Convert a protocol buffer Filter to a client Filter
1465    *
1466    * @param proto the protocol buffer Filter to convert
1467    * @return the converted Filter
1468    */
1469   @SuppressWarnings("unchecked")
1470   public static Filter toFilter(FilterProtos.Filter proto) throws IOException {
1471     String type = proto.getName();
1472     final byte [] value = proto.getSerializedFilter().toByteArray();
1473     String funcName = "parseFrom";
1474     try {
1475       Class<? extends Filter> c =
1476         (Class<? extends Filter>)Class.forName(type, true, CLASS_LOADER);
1477       Method parseFrom = c.getMethod(funcName, byte[].class);
1478       if (parseFrom == null) {
1479         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1480       }
1481       return (Filter)parseFrom.invoke(c, value);
1482     } catch (Exception e) {
1483       // Either we couldn't instantiate the method object, or "parseFrom" failed.
1484       // In either case, let's not retry.
1485       throw new DoNotRetryIOException(e);
1486     }
1487   }
1488
1489   /**
1490    * Convert a client Filter to a protocol buffer Filter
1491    *
1492    * @param filter the Filter to convert
1493    * @return the converted protocol buffer Filter
1494    */
1495   public static FilterProtos.Filter toFilter(Filter filter) throws IOException {
1496     FilterProtos.Filter.Builder builder = FilterProtos.Filter.newBuilder();
1497     builder.setName(filter.getClass().getName());
1498     builder.setSerializedFilter(ByteStringer.wrap(filter.toByteArray()));
1499     return builder.build();
1500   }
1501
1502   /**
1503    * Convert a delete KeyValue type to protocol buffer DeleteType.
1504    *
1505    * @param type
1506    * @return protocol buffer DeleteType
1507    * @throws IOException
1508    */
1509   public static DeleteType toDeleteType(
1510       KeyValue.Type type) throws IOException {
1511     switch (type) {
1512     case Delete:
1513       return DeleteType.DELETE_ONE_VERSION;
1514     case DeleteColumn:
1515       return DeleteType.DELETE_MULTIPLE_VERSIONS;
1516     case DeleteFamily:
1517       return DeleteType.DELETE_FAMILY;
1518     case DeleteFamilyVersion:
1519       return DeleteType.DELETE_FAMILY_VERSION;
1520     default:
1521         throw new IOException("Unknown delete type: " + type);
1522     }
1523   }
1524
1525   /**
1526    * Convert a protocol buffer DeleteType to delete KeyValue type.
1527    *
1528    * @param type The DeleteType
1529    * @return The type.
1530    * @throws IOException
1531    */
1532   public static KeyValue.Type fromDeleteType(
1533       DeleteType type) throws IOException {
1534     switch (type) {
1535     case DELETE_ONE_VERSION:
1536       return KeyValue.Type.Delete;
1537     case DELETE_MULTIPLE_VERSIONS:
1538       return KeyValue.Type.DeleteColumn;
1539     case DELETE_FAMILY:
1540       return KeyValue.Type.DeleteFamily;
1541     case DELETE_FAMILY_VERSION:
1542       return KeyValue.Type.DeleteFamilyVersion;
1543     default:
1544       throw new IOException("Unknown delete type: " + type);
1545     }
1546   }
1547
1548   /**
1549    * Convert a stringified protocol buffer exception Parameter to a Java Exception
1550    *
1551    * @param parameter the protocol buffer Parameter to convert
1552    * @return the converted Exception
1553    * @throws IOException if failed to deserialize the parameter
1554    */
1555   @SuppressWarnings("unchecked")
1556   public static Throwable toException(final NameBytesPair parameter) throws IOException {
1557     if (parameter == null || !parameter.hasValue()) return null;
1558     String desc = parameter.getValue().toStringUtf8();
1559     String type = parameter.getName();
1560     try {
1561       Class<? extends Throwable> c =
1562         (Class<? extends Throwable>)Class.forName(type, true, CLASS_LOADER);
1563       Constructor<? extends Throwable> cn = null;
1564       try {
1565         cn = c.getDeclaredConstructor(String.class);
1566         return cn.newInstance(desc);
1567       } catch (NoSuchMethodException e) {
1568         // Could be a raw RemoteException. See HBASE-8987.
1569         cn = c.getDeclaredConstructor(String.class, String.class);
1570         return cn.newInstance(type, desc);
1571       }
1572     } catch (Exception e) {
1573       throw new IOException(e);
1574     }
1575   }
1576
1577 // Start helpers for Client
1578
1579   public static CoprocessorServiceResponse execService(final RpcController controller,
1580       final ClientService.BlockingInterface client, final CoprocessorServiceCall call,
1581       final byte[] regionName) throws IOException {
1582     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1583         .setCall(call).setRegion(
1584             RequestConverter.buildRegionSpecifier(REGION_NAME, regionName)).build();
1585     try {
1586       CoprocessorServiceResponse response =
1587           client.execService(controller, request);
1588       return response;
1589     } catch (ServiceException se) {
1590       throw getRemoteException(se);
1591     }
1592   }
1593
1594   public static CoprocessorServiceResponse execService(final RpcController controller,
1595       final MasterService.BlockingInterface client, final CoprocessorServiceCall call)
1596       throws IOException {
1597     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1598         .setCall(call).setRegion(
1599            RequestConverter.buildRegionSpecifier(REGION_NAME, HConstants.EMPTY_BYTE_ARRAY)).build();
1600     try {
1601       CoprocessorServiceResponse response =
1602           client.execMasterService(controller, request);
1603       return response;
1604     } catch (ServiceException se) {
1605       throw getRemoteException(se);
1606     }
1607   }
1608
1609   /**
1610    * Make a region server endpoint call
1611    * @param client
1612    * @param call
1613    * @return CoprocessorServiceResponse
1614    * @throws IOException
1615    */
1616   public static CoprocessorServiceResponse execRegionServerService(
1617       final RpcController controller, final ClientService.BlockingInterface client,
1618       final CoprocessorServiceCall call)
1619       throws IOException {
1620     CoprocessorServiceRequest request =
1621         CoprocessorServiceRequest
1622             .newBuilder()
1623             .setCall(call)
1624             .setRegion(
1625               RequestConverter.buildRegionSpecifier(REGION_NAME, HConstants.EMPTY_BYTE_ARRAY))
1626             .build();
1627     try {
1628       CoprocessorServiceResponse response = client.execRegionServerService(controller, request);
1629       return response;
1630     } catch (ServiceException se) {
1631       throw getRemoteException(se);
1632     }
1633   }
1634
1635   @SuppressWarnings("unchecked")
1636   public static <T extends Service> T newServiceStub(Class<T> service, RpcChannel channel)
1637       throws Exception {
1638     return (T)Methods.call(service, null, "newStub",
1639         new Class[]{ RpcChannel.class }, new Object[]{ channel });
1640   }
1641
1642 // End helpers for Client
1643 // Start helpers for Admin
1644
1645   /**
1646    * A helper to retrieve region info given a region name
1647    * using admin protocol.
1648    *
1649    * @param admin
1650    * @param regionName
1651    * @return the retrieved region info
1652    * @throws IOException
1653    */
1654   public static HRegionInfo getRegionInfo(final RpcController controller,
1655       final AdminService.BlockingInterface admin, final byte[] regionName) throws IOException {
1656     try {
1657       GetRegionInfoRequest request =
1658         RequestConverter.buildGetRegionInfoRequest(regionName);
1659       GetRegionInfoResponse response =
1660         admin.getRegionInfo(controller, request);
1661       return HRegionInfo.convert(response.getRegionInfo());
1662     } catch (ServiceException se) {
1663       throw getRemoteException(se);
1664     }
1665   }
1666
1667   /**
1668    * A helper to close a region given a region name
1669    * using admin protocol.
1670    *
1671    * @param admin
1672    * @param regionName
1673    * @throws IOException
1674    */
1675   public static void closeRegion(final RpcController controller,
1676       final AdminService.BlockingInterface admin, final ServerName server, final byte[] regionName)
1677           throws IOException {
1678     CloseRegionRequest closeRegionRequest =
1679       RequestConverter.buildCloseRegionRequest(server, regionName);
1680     try {
1681       admin.closeRegion(controller, closeRegionRequest);
1682     } catch (ServiceException se) {
1683       throw getRemoteException(se);
1684     }
1685   }
1686
1687   /**
1688    * A helper to close a region given a region name
1689    * using admin protocol.
1690    *
1691    * @param admin
1692    * @param regionName
1693    * @return true if the region is closed
1694    * @throws IOException
1695    */
1696   public static boolean closeRegion(final RpcController controller,
1697       final AdminService.BlockingInterface admin,
1698       final ServerName server, final byte[] regionName,
1699       final ServerName destinationServer) throws IOException {
1700     CloseRegionRequest closeRegionRequest =
1701       RequestConverter.buildCloseRegionRequest(server,
1702         regionName, destinationServer);
1703     try {
1704       CloseRegionResponse response = admin.closeRegion(controller, closeRegionRequest);
1705       return ResponseConverter.isClosed(response);
1706     } catch (ServiceException se) {
1707       throw getRemoteException(se);
1708     }
1709   }
1710
1711   /**
1712    * A helper to warmup a region given a region name
1713    * using admin protocol
1714    *
1715    * @param admin
1716    * @param regionInfo
1717    *
1718    */
1719   public static void warmupRegion(final RpcController controller,
1720       final AdminService.BlockingInterface admin, final HRegionInfo regionInfo) throws IOException {
1721
1722     try {
1723       WarmupRegionRequest warmupRegionRequest =
1724            RequestConverter.buildWarmupRegionRequest(regionInfo);
1725
1726       admin.warmupRegion(controller, warmupRegionRequest);
1727     } catch (ServiceException e) {
1728       throw getRemoteException(e);
1729     }
1730   }
1731
1732   /**
1733    * A helper to open a region using admin protocol.
1734    * @param admin
1735    * @param region
1736    * @throws IOException
1737    */
1738   public static void openRegion(final RpcController controller,
1739       final AdminService.BlockingInterface admin, ServerName server, final HRegionInfo region)
1740           throws IOException {
1741     OpenRegionRequest request =
1742       RequestConverter.buildOpenRegionRequest(server, region, null, null);
1743     try {
1744       admin.openRegion(controller, request);
1745     } catch (ServiceException se) {
1746       throw ProtobufUtil.getRemoteException(se);
1747     }
1748   }
1749
1750   /**
1751    * A helper to get the all the online regions on a region
1752    * server using admin protocol.
1753    *
1754    * @param admin
1755    * @return a list of online region info
1756    * @throws IOException
1757    */
1758   public static List<HRegionInfo> getOnlineRegions(final AdminService.BlockingInterface admin)
1759       throws IOException {
1760     return getOnlineRegions(null, admin);
1761   }
1762
1763   /**
1764    * A helper to get the all the online regions on a region
1765    * server using admin protocol.
1766    * @return a list of online region info
1767    */
1768   public static List<HRegionInfo> getOnlineRegions(final RpcController controller,
1769       final AdminService.BlockingInterface admin)
1770   throws IOException {
1771     GetOnlineRegionRequest request = RequestConverter.buildGetOnlineRegionRequest();
1772     GetOnlineRegionResponse response = null;
1773     try {
1774       response = admin.getOnlineRegion(controller, request);
1775     } catch (ServiceException se) {
1776       throw getRemoteException(se);
1777     }
1778     return getRegionInfos(response);
1779   }
1780
1781   /**
1782    * Get the list of region info from a GetOnlineRegionResponse
1783    *
1784    * @param proto the GetOnlineRegionResponse
1785    * @return the list of region info or null if <code>proto</code> is null
1786    */
1787   static List<HRegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
1788     if (proto == null) return null;
1789     List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
1790     for (RegionInfo regionInfo: proto.getRegionInfoList()) {
1791       regionInfos.add(HRegionInfo.convert(regionInfo));
1792     }
1793     return regionInfos;
1794   }
1795
1796   /**
1797    * A helper to get the info of a region server using admin protocol.
1798    * @return the server name
1799    */
1800   public static ServerInfo getServerInfo(final RpcController controller,
1801       final AdminService.BlockingInterface admin)
1802   throws IOException {
1803     GetServerInfoRequest request = RequestConverter.buildGetServerInfoRequest();
1804     try {
1805       GetServerInfoResponse response = admin.getServerInfo(controller, request);
1806       return response.getServerInfo();
1807     } catch (ServiceException se) {
1808       throw getRemoteException(se);
1809     }
1810   }
1811
1812   /**
1813    * A helper to get the list of files of a column family
1814    * on a given region using admin protocol.
1815    *
1816    * @return the list of store files
1817    */
1818   public static List<String> getStoreFiles(final AdminService.BlockingInterface admin,
1819       final byte[] regionName, final byte[] family)
1820   throws IOException {
1821     return getStoreFiles(null, admin, regionName, family);
1822   }
1823
1824   /**
1825    * A helper to get the list of files of a column family
1826    * on a given region using admin protocol.
1827    *
1828    * @return the list of store files
1829    */
1830   public static List<String> getStoreFiles(final RpcController controller,
1831       final AdminService.BlockingInterface admin, final byte[] regionName, final byte[] family)
1832   throws IOException {
1833     GetStoreFileRequest request =
1834       RequestConverter.buildGetStoreFileRequest(regionName, family);
1835     try {
1836       GetStoreFileResponse response = admin.getStoreFile(controller, request);
1837       return response.getStoreFileList();
1838     } catch (ServiceException se) {
1839       throw ProtobufUtil.getRemoteException(se);
1840     }
1841   }
1842
1843   /**
1844    * A helper to split a region using admin protocol.
1845    *
1846    * @param admin
1847    * @param hri
1848    * @param splitPoint
1849    * @throws IOException
1850    */
1851   public static void split(final RpcController controller,
1852       final AdminService.BlockingInterface admin, final HRegionInfo hri, byte[] splitPoint)
1853           throws IOException {
1854     SplitRegionRequest request =
1855       RequestConverter.buildSplitRegionRequest(hri.getRegionName(), splitPoint);
1856     try {
1857       admin.splitRegion(controller, request);
1858     } catch (ServiceException se) {
1859       throw ProtobufUtil.getRemoteException(se);
1860     }
1861   }
1862
1863   /**
1864    * A helper to merge regions using admin protocol. Send request to
1865    * regionserver.
1866    * @param admin
1867    * @param region_a
1868    * @param region_b
1869    * @param forcible true if do a compulsory merge, otherwise we will only merge
1870    *          two adjacent regions
1871    * @param user effective user
1872    * @throws IOException
1873    */
1874   public static void mergeRegions(final RpcController controller,
1875       final AdminService.BlockingInterface admin,
1876       final HRegionInfo region_a, final HRegionInfo region_b,
1877       final boolean forcible, final UserGroupInformation user) throws IOException {
1878     final MergeRegionsRequest request = RequestConverter.buildMergeRegionsRequest(
1879         region_a.getRegionName(), region_b.getRegionName(),forcible);
1880     if (user != null) {
1881       try {
1882         user.doAs(new PrivilegedExceptionAction<Void>() {
1883           @Override
1884           public Void run() throws Exception {
1885             admin.mergeRegions(controller, request);
1886             return null;
1887           }
1888         });
1889       } catch (InterruptedException ie) {
1890         InterruptedIOException iioe = new InterruptedIOException();
1891         iioe.initCause(ie);
1892         throw iioe;
1893       }
1894     } else {
1895       try {
1896         admin.mergeRegions(controller, request);
1897       } catch (ServiceException se) {
1898         throw ProtobufUtil.getRemoteException(se);
1899       }
1900     }
1901   }
1902
1903 // End helpers for Admin
1904
1905   /*
1906    * Get the total (read + write) requests from a RegionLoad pb
1907    * @param rl - RegionLoad pb
1908    * @return total (read + write) requests
1909    */
1910   public static long getTotalRequestsCount(RegionLoad rl) {
1911     if (rl == null) {
1912       return 0;
1913     }
1914
1915     return rl.getReadRequestsCount() + rl.getWriteRequestsCount();
1916   }
1917
1918
1919   /**
1920    * @param m Message to get delimited pb serialization of (with pb magic prefix)
1921    */
1922   public static byte [] toDelimitedByteArray(final Message m) throws IOException {
1923     // Allocate arbitrary big size so we avoid resizing.
1924     ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
1925     baos.write(ProtobufMagic.PB_MAGIC);
1926     m.writeDelimitedTo(baos);
1927     return baos.toByteArray();
1928   }
1929
1930   /**
1931    * Converts a Permission proto to a client Permission object.
1932    *
1933    * @param proto the protobuf Permission
1934    * @return the converted Permission
1935    */
1936   public static Permission toPermission(AccessControlProtos.Permission proto) {
1937     if (proto.getType() != AccessControlProtos.Permission.Type.Global) {
1938       return toTablePermission(proto);
1939     } else {
1940       List<Permission.Action> actions = toPermissionActions(
1941         proto.getGlobalPermission().getActionList());
1942       return new Permission(actions.toArray(new Permission.Action[actions.size()]));
1943     }
1944   }
1945
1946   /**
1947    * Converts a Permission proto to a client TablePermission object.
1948    *
1949    * @param proto the protobuf Permission
1950    * @return the converted TablePermission
1951    */
1952   public static TablePermission toTablePermission(AccessControlProtos.Permission proto) {
1953     if(proto.getType() == AccessControlProtos.Permission.Type.Global) {
1954       AccessControlProtos.GlobalPermission perm = proto.getGlobalPermission();
1955       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1956
1957       return new TablePermission(null, null, null,
1958           actions.toArray(new Permission.Action[actions.size()]));
1959     }
1960     if(proto.getType() == AccessControlProtos.Permission.Type.Namespace) {
1961       AccessControlProtos.NamespacePermission perm = proto.getNamespacePermission();
1962       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1963
1964       if(!proto.hasNamespacePermission()) {
1965         throw new IllegalStateException("Namespace must not be empty in NamespacePermission");
1966       }
1967       String namespace = perm.getNamespaceName().toStringUtf8();
1968       return new TablePermission(namespace, actions.toArray(new Permission.Action[actions.size()]));
1969     }
1970     if(proto.getType() == AccessControlProtos.Permission.Type.Table) {
1971       AccessControlProtos.TablePermission perm = proto.getTablePermission();
1972       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1973
1974       byte[] qualifier = null;
1975       byte[] family = null;
1976       TableName table = null;
1977
1978       if (!perm.hasTableName()) {
1979         throw new IllegalStateException("TableName cannot be empty");
1980       }
1981       table = ProtobufUtil.toTableName(perm.getTableName());
1982
1983       if (perm.hasFamily()) family = perm.getFamily().toByteArray();
1984       if (perm.hasQualifier()) qualifier = perm.getQualifier().toByteArray();
1985
1986       return new TablePermission(table, family, qualifier,
1987           actions.toArray(new Permission.Action[actions.size()]));
1988     }
1989     throw new IllegalStateException("Unrecognize Perm Type: "+proto.getType());
1990   }
1991
1992   /**
1993    * Convert a client Permission to a Permission proto
1994    *
1995    * @param perm the client Permission
1996    * @return the protobuf Permission
1997    */
1998   public static AccessControlProtos.Permission toPermission(Permission perm) {
1999     AccessControlProtos.Permission.Builder ret = AccessControlProtos.Permission.newBuilder();
2000     if (perm instanceof TablePermission) {
2001       TablePermission tablePerm = (TablePermission)perm;
2002       if(tablePerm.hasNamespace()) {
2003         ret.setType(AccessControlProtos.Permission.Type.Namespace);
2004
2005         AccessControlProtos.NamespacePermission.Builder builder =
2006             AccessControlProtos.NamespacePermission.newBuilder();
2007         builder.setNamespaceName(ByteString.copyFromUtf8(tablePerm.getNamespace()));
2008         Permission.Action[] actions = perm.getActions();
2009         if (actions != null) {
2010           for (Permission.Action a : actions) {
2011             builder.addAction(toPermissionAction(a));
2012           }
2013         }
2014         ret.setNamespacePermission(builder);
2015         return ret.build();
2016       } else if (tablePerm.hasTable()) {
2017         ret.setType(AccessControlProtos.Permission.Type.Table);
2018
2019         AccessControlProtos.TablePermission.Builder builder =
2020             AccessControlProtos.TablePermission.newBuilder();
2021         builder.setTableName(ProtobufUtil.toProtoTableName(tablePerm.getTableName()));
2022         if (tablePerm.hasFamily()) {
2023           builder.setFamily(ByteStringer.wrap(tablePerm.getFamily()));
2024         }
2025         if (tablePerm.hasQualifier()) {
2026           builder.setQualifier(ByteStringer.wrap(tablePerm.getQualifier()));
2027         }
2028         Permission.Action actions[] = perm.getActions();
2029         if (actions != null) {
2030           for (Permission.Action a : actions) {
2031             builder.addAction(toPermissionAction(a));
2032           }
2033         }
2034         ret.setTablePermission(builder);
2035         return ret.build();
2036       }
2037     }
2038
2039     ret.setType(AccessControlProtos.Permission.Type.Global);
2040
2041     AccessControlProtos.GlobalPermission.Builder builder =
2042         AccessControlProtos.GlobalPermission.newBuilder();
2043     Permission.Action actions[] = perm.getActions();
2044     if (actions != null) {
2045       for (Permission.Action a: actions) {
2046         builder.addAction(toPermissionAction(a));
2047       }
2048     }
2049     ret.setGlobalPermission(builder);
2050     return ret.build();
2051   }
2052
2053   /**
2054    * Converts a list of Permission.Action proto to a list of client Permission.Action objects.
2055    *
2056    * @param protoActions the list of protobuf Actions
2057    * @return the converted list of Actions
2058    */
2059   public static List<Permission.Action> toPermissionActions(
2060       List<AccessControlProtos.Permission.Action> protoActions) {
2061     List<Permission.Action> actions = new ArrayList<Permission.Action>(protoActions.size());
2062     for (AccessControlProtos.Permission.Action a : protoActions) {
2063       actions.add(toPermissionAction(a));
2064     }
2065     return actions;
2066   }
2067
2068   /**
2069    * Converts a Permission.Action proto to a client Permission.Action object.
2070    *
2071    * @param action the protobuf Action
2072    * @return the converted Action
2073    */
2074   public static Permission.Action toPermissionAction(
2075       AccessControlProtos.Permission.Action action) {
2076     switch (action) {
2077       case READ:
2078         return Permission.Action.READ;
2079       case WRITE:
2080         return Permission.Action.WRITE;
2081       case EXEC:
2082         return Permission.Action.EXEC;
2083       case CREATE:
2084         return Permission.Action.CREATE;
2085       case ADMIN:
2086         return Permission.Action.ADMIN;
2087     }
2088     throw new IllegalArgumentException("Unknown action value "+action.name());
2089   }
2090
2091   /**
2092    * Convert a client Permission.Action to a Permission.Action proto
2093    *
2094    * @param action the client Action
2095    * @return the protobuf Action
2096    */
2097   public static AccessControlProtos.Permission.Action toPermissionAction(
2098       Permission.Action action) {
2099     switch (action) {
2100       case READ:
2101         return AccessControlProtos.Permission.Action.READ;
2102       case WRITE:
2103         return AccessControlProtos.Permission.Action.WRITE;
2104       case EXEC:
2105         return AccessControlProtos.Permission.Action.EXEC;
2106       case CREATE:
2107         return AccessControlProtos.Permission.Action.CREATE;
2108       case ADMIN:
2109         return AccessControlProtos.Permission.Action.ADMIN;
2110     }
2111     throw new IllegalArgumentException("Unknown action value "+action.name());
2112   }
2113
2114   /**
2115    * Convert a client user permission to a user permission proto
2116    *
2117    * @param perm the client UserPermission
2118    * @return the protobuf UserPermission
2119    */
2120   public static AccessControlProtos.UserPermission toUserPermission(UserPermission perm) {
2121     return AccessControlProtos.UserPermission.newBuilder()
2122         .setUser(ByteStringer.wrap(perm.getUser()))
2123         .setPermission(toPermission(perm))
2124         .build();
2125   }
2126
2127   /**
2128    * Converts a user permission proto to a client user permission object.
2129    *
2130    * @param proto the protobuf UserPermission
2131    * @return the converted UserPermission
2132    */
2133   public static UserPermission toUserPermission(AccessControlProtos.UserPermission proto) {
2134     return new UserPermission(proto.getUser().toByteArray(),
2135         toTablePermission(proto.getPermission()));
2136   }
2137
2138   /**
2139    * Convert a ListMultimap&lt;String, TablePermission&gt; where key is username
2140    * to a protobuf UserPermission
2141    *
2142    * @param perm the list of user and table permissions
2143    * @return the protobuf UserTablePermissions
2144    */
2145   public static AccessControlProtos.UsersAndPermissions toUserTablePermissions(
2146       ListMultimap<String, TablePermission> perm) {
2147     AccessControlProtos.UsersAndPermissions.Builder builder =
2148                   AccessControlProtos.UsersAndPermissions.newBuilder();
2149     for (Map.Entry<String, Collection<TablePermission>> entry : perm.asMap().entrySet()) {
2150       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
2151                   AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
2152       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
2153       for (TablePermission tablePerm: entry.getValue()) {
2154         userPermBuilder.addPermissions(toPermission(tablePerm));
2155       }
2156       builder.addUserPermissions(userPermBuilder.build());
2157     }
2158     return builder.build();
2159   }
2160
2161   /**
2162    * A utility used to grant a user global permissions.
2163    * <p>
2164    * It's also called by the shell, in case you want to find references.
2165    *
2166    * @param protocol the AccessControlService protocol proxy
2167    * @param userShortName the short name of the user to grant permissions
2168    * @param actions the permissions to be granted
2169    * @throws ServiceException
2170    */
2171   public static void grant(RpcController controller,
2172       AccessControlService.BlockingInterface protocol, String userShortName,
2173       Permission.Action... actions) throws ServiceException {
2174     List<AccessControlProtos.Permission.Action> permActions =
2175         Lists.newArrayListWithCapacity(actions.length);
2176     for (Permission.Action a : actions) {
2177       permActions.add(ProtobufUtil.toPermissionAction(a));
2178     }
2179     AccessControlProtos.GrantRequest request = RequestConverter.
2180       buildGrantRequest(userShortName, permActions.toArray(
2181         new AccessControlProtos.Permission.Action[actions.length]));
2182     protocol.grant(controller, request);
2183   }
2184
2185   /**
2186    * A utility used to grant a user table permissions. The permissions will
2187    * be for a table table/column family/qualifier.
2188    * <p>
2189    * It's also called by the shell, in case you want to find references.
2190    *
2191    * @param protocol the AccessControlService protocol proxy
2192    * @param userShortName the short name of the user to grant permissions
2193    * @param tableName optional table name
2194    * @param f optional column family
2195    * @param q optional qualifier
2196    * @param actions the permissions to be granted
2197    * @throws ServiceException
2198    */
2199   public static void grant(RpcController controller,
2200       AccessControlService.BlockingInterface protocol, String userShortName, TableName tableName,
2201       byte[] f, byte[] q, Permission.Action... actions) throws ServiceException {
2202     List<AccessControlProtos.Permission.Action> permActions =
2203         Lists.newArrayListWithCapacity(actions.length);
2204     for (Permission.Action a : actions) {
2205       permActions.add(ProtobufUtil.toPermissionAction(a));
2206     }
2207     AccessControlProtos.GrantRequest request = RequestConverter.
2208       buildGrantRequest(userShortName, tableName, f, q, permActions.toArray(
2209         new AccessControlProtos.Permission.Action[actions.length]));
2210     protocol.grant(controller, request);
2211   }
2212
2213   /**
2214    * A utility used to grant a user namespace permissions.
2215    * <p>
2216    * It's also called by the shell, in case you want to find references.
2217    *
2218    * @param protocol the AccessControlService protocol proxy
2219    * @param namespace the short name of the user to grant permissions
2220    * @param actions the permissions to be granted
2221    * @throws ServiceException
2222    */
2223   public static void grant(RpcController controller,
2224       AccessControlService.BlockingInterface protocol, String userShortName, String namespace,
2225       Permission.Action... actions) throws ServiceException {
2226     List<AccessControlProtos.Permission.Action> permActions =
2227         Lists.newArrayListWithCapacity(actions.length);
2228     for (Permission.Action a : actions) {
2229       permActions.add(ProtobufUtil.toPermissionAction(a));
2230     }
2231     AccessControlProtos.GrantRequest request = RequestConverter.
2232       buildGrantRequest(userShortName, namespace, permActions.toArray(
2233         new AccessControlProtos.Permission.Action[actions.length]));
2234     protocol.grant(controller, request);
2235   }
2236
2237   /**
2238    * A utility used to revoke a user's global permissions.
2239    * <p>
2240    * It's also called by the shell, in case you want to find references.
2241    *
2242    * @param protocol the AccessControlService protocol proxy
2243    * @param userShortName the short name of the user to revoke permissions
2244    * @param actions the permissions to be revoked
2245    * @throws ServiceException
2246    */
2247   public static void revoke(RpcController controller,
2248       AccessControlService.BlockingInterface protocol, String userShortName,
2249       Permission.Action... actions) throws ServiceException {
2250     List<AccessControlProtos.Permission.Action> permActions =
2251         Lists.newArrayListWithCapacity(actions.length);
2252     for (Permission.Action a : actions) {
2253       permActions.add(ProtobufUtil.toPermissionAction(a));
2254     }
2255     AccessControlProtos.RevokeRequest request = RequestConverter.
2256       buildRevokeRequest(userShortName, permActions.toArray(
2257         new AccessControlProtos.Permission.Action[actions.length]));
2258     protocol.revoke(controller, request);
2259   }
2260
2261   /**
2262    * A utility used to revoke a user's table permissions. The permissions will
2263    * be for a table/column family/qualifier.
2264    * <p>
2265    * It's also called by the shell, in case you want to find references.
2266    *
2267    * @param protocol the AccessControlService protocol proxy
2268    * @param userShortName the short name of the user to revoke permissions
2269    * @param tableName optional table name
2270    * @param f optional column family
2271    * @param q optional qualifier
2272    * @param actions the permissions to be revoked
2273    * @throws ServiceException
2274    */
2275   public static void revoke(RpcController controller,
2276       AccessControlService.BlockingInterface protocol, String userShortName, TableName tableName,
2277       byte[] f, byte[] q, Permission.Action... actions) throws ServiceException {
2278     List<AccessControlProtos.Permission.Action> permActions =
2279         Lists.newArrayListWithCapacity(actions.length);
2280     for (Permission.Action a : actions) {
2281       permActions.add(ProtobufUtil.toPermissionAction(a));
2282     }
2283     AccessControlProtos.RevokeRequest request = RequestConverter.
2284       buildRevokeRequest(userShortName, tableName, f, q, permActions.toArray(
2285         new AccessControlProtos.Permission.Action[actions.length]));
2286     protocol.revoke(controller, request);
2287   }
2288
2289   /**
2290    * A utility used to revoke a user's namespace permissions.
2291    * <p>
2292    * It's also called by the shell, in case you want to find references.
2293    *
2294    * @param protocol the AccessControlService protocol proxy
2295    * @param userShortName the short name of the user to revoke permissions
2296    * @param namespace optional table name
2297    * @param actions the permissions to be revoked
2298    * @throws ServiceException
2299    */
2300   public static void revoke(RpcController controller,
2301       AccessControlService.BlockingInterface protocol, String userShortName, String namespace,
2302       Permission.Action... actions) throws ServiceException {
2303     List<AccessControlProtos.Permission.Action> permActions =
2304         Lists.newArrayListWithCapacity(actions.length);
2305     for (Permission.Action a : actions) {
2306       permActions.add(ProtobufUtil.toPermissionAction(a));
2307     }
2308     AccessControlProtos.RevokeRequest request = RequestConverter.
2309       buildRevokeRequest(userShortName, namespace, permActions.toArray(
2310         new AccessControlProtos.Permission.Action[actions.length]));
2311     protocol.revoke(controller, request);
2312   }
2313
2314   /**
2315    * A utility used to get user's global permissions.
2316    * <p>
2317    * It's also called by the shell, in case you want to find references.
2318    *
2319    * @param protocol the AccessControlService protocol proxy
2320    * @throws ServiceException
2321    */
2322   public static List<UserPermission> getUserPermissions(RpcController controller,
2323       AccessControlService.BlockingInterface protocol) throws ServiceException {
2324     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2325       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2326     builder.setType(AccessControlProtos.Permission.Type.Global);
2327     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2328     AccessControlProtos.GetUserPermissionsResponse response =
2329       protocol.getUserPermissions(controller, request);
2330     List<UserPermission> perms = new ArrayList<UserPermission>(response.getUserPermissionCount());
2331     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2332       perms.add(ProtobufUtil.toUserPermission(perm));
2333     }
2334     return perms;
2335   }
2336
2337   /**
2338    * A utility used to get user table permissions.
2339    * <p>
2340    * It's also called by the shell, in case you want to find references.
2341    *
2342    * @param protocol the AccessControlService protocol proxy
2343    * @param t optional table name
2344    * @throws ServiceException
2345    */
2346   public static List<UserPermission> getUserPermissions(RpcController controller,
2347       AccessControlService.BlockingInterface protocol,
2348       TableName t) throws ServiceException {
2349     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2350       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2351     if (t != null) {
2352       builder.setTableName(ProtobufUtil.toProtoTableName(t));
2353     }
2354     builder.setType(AccessControlProtos.Permission.Type.Table);
2355     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2356     AccessControlProtos.GetUserPermissionsResponse response =
2357       protocol.getUserPermissions(controller, request);
2358     List<UserPermission> perms = new ArrayList<UserPermission>(response.getUserPermissionCount());
2359     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2360       perms.add(ProtobufUtil.toUserPermission(perm));
2361     }
2362     return perms;
2363   }
2364
2365   /**
2366    * A utility used to get permissions for selected namespace.
2367    * <p>
2368    * It's also called by the shell, in case you want to find references.
2369    *
2370    * @param protocol the AccessControlService protocol proxy
2371    * @param namespace name of the namespace
2372    * @throws ServiceException
2373    */
2374   public static List<UserPermission> getUserPermissions(RpcController controller,
2375       AccessControlService.BlockingInterface protocol,
2376       byte[] namespace) throws ServiceException {
2377     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2378       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2379     if (namespace != null) {
2380       builder.setNamespaceName(ByteStringer.wrap(namespace));
2381     }
2382     builder.setType(AccessControlProtos.Permission.Type.Namespace);
2383     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2384     AccessControlProtos.GetUserPermissionsResponse response =
2385       protocol.getUserPermissions(controller, request);
2386     List<UserPermission> perms = new ArrayList<UserPermission>(response.getUserPermissionCount());
2387     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2388       perms.add(ProtobufUtil.toUserPermission(perm));
2389     }
2390     return perms;
2391   }
2392
2393   /**
2394    * Convert a protobuf UserTablePermissions to a
2395    * ListMultimap&lt;String, TablePermission&gt; where key is username.
2396    *
2397    * @param proto the protobuf UserPermission
2398    * @return the converted UserPermission
2399    */
2400   public static ListMultimap<String, TablePermission> toUserTablePermissions(
2401       AccessControlProtos.UsersAndPermissions proto) {
2402     ListMultimap<String, TablePermission> perms = ArrayListMultimap.create();
2403     AccessControlProtos.UsersAndPermissions.UserPermissions userPerm;
2404
2405     for (int i = 0; i < proto.getUserPermissionsCount(); i++) {
2406       userPerm = proto.getUserPermissions(i);
2407       for (int j = 0; j < userPerm.getPermissionsCount(); j++) {
2408         TablePermission tablePerm = toTablePermission(userPerm.getPermissions(j));
2409         perms.put(userPerm.getUser().toStringUtf8(), tablePerm);
2410       }
2411     }
2412
2413     return perms;
2414   }
2415
2416   /**
2417    * Converts a Token instance (with embedded identifier) to the protobuf representation.
2418    *
2419    * @param token the Token instance to copy
2420    * @return the protobuf Token message
2421    */
2422   public static AuthenticationProtos.Token toToken(Token<AuthenticationTokenIdentifier> token) {
2423     AuthenticationProtos.Token.Builder builder = AuthenticationProtos.Token.newBuilder();
2424     builder.setIdentifier(ByteStringer.wrap(token.getIdentifier()));
2425     builder.setPassword(ByteStringer.wrap(token.getPassword()));
2426     if (token.getService() != null) {
2427       builder.setService(ByteString.copyFromUtf8(token.getService().toString()));
2428     }
2429     return builder.build();
2430   }
2431
2432   /**
2433    * Converts a protobuf Token message back into a Token instance.
2434    *
2435    * @param proto the protobuf Token message
2436    * @return the Token instance
2437    */
2438   public static Token<AuthenticationTokenIdentifier> toToken(AuthenticationProtos.Token proto) {
2439     return new Token<AuthenticationTokenIdentifier>(
2440         proto.hasIdentifier() ? proto.getIdentifier().toByteArray() : null,
2441         proto.hasPassword() ? proto.getPassword().toByteArray() : null,
2442         AuthenticationTokenIdentifier.AUTH_TOKEN_TYPE,
2443         proto.hasService() ? new Text(proto.getService().toStringUtf8()) : null);
2444   }
2445
2446   /**
2447    * Find the HRegion encoded name based on a region specifier
2448    *
2449    * @param regionSpecifier the region specifier
2450    * @return the corresponding region's encoded name
2451    * @throws DoNotRetryIOException if the specifier type is unsupported
2452    */
2453   public static String getRegionEncodedName(
2454       final RegionSpecifier regionSpecifier) throws DoNotRetryIOException {
2455     ByteString value = regionSpecifier.getValue();
2456     RegionSpecifierType type = regionSpecifier.getType();
2457     switch (type) {
2458       case REGION_NAME:
2459         return HRegionInfo.encodeRegionName(value.toByteArray());
2460       case ENCODED_REGION_NAME:
2461         return value.toStringUtf8();
2462       default:
2463         throw new DoNotRetryIOException(
2464           "Unsupported region specifier type: " + type);
2465     }
2466   }
2467
2468   public static ScanMetrics toScanMetrics(final byte[] bytes) {
2469     Parser<MapReduceProtos.ScanMetrics> parser = MapReduceProtos.ScanMetrics.PARSER;
2470     MapReduceProtos.ScanMetrics pScanMetrics = null;
2471     try {
2472       pScanMetrics = parser.parseFrom(bytes);
2473     } catch (InvalidProtocolBufferException e) {
2474       //Ignored there are just no key values to add.
2475     }
2476     ScanMetrics scanMetrics = new ScanMetrics();
2477     if (pScanMetrics != null) {
2478       for (HBaseProtos.NameInt64Pair pair : pScanMetrics.getMetricsList()) {
2479         if (pair.hasName() && pair.hasValue()) {
2480           scanMetrics.setCounter(pair.getName(), pair.getValue());
2481         }
2482       }
2483     }
2484     return scanMetrics;
2485   }
2486
2487   public static MapReduceProtos.ScanMetrics toScanMetrics(ScanMetrics scanMetrics) {
2488     MapReduceProtos.ScanMetrics.Builder builder = MapReduceProtos.ScanMetrics.newBuilder();
2489     Map<String, Long> metrics = scanMetrics.getMetricsMap();
2490     for (Entry<String, Long> e : metrics.entrySet()) {
2491       HBaseProtos.NameInt64Pair nameInt64Pair =
2492           HBaseProtos.NameInt64Pair.newBuilder()
2493               .setName(e.getKey())
2494               .setValue(e.getValue())
2495               .build();
2496       builder.addMetrics(nameInt64Pair);
2497     }
2498     return builder.build();
2499   }
2500
2501   /**
2502    * Unwraps an exception from a protobuf service into the underlying (expected) IOException.
2503    * This method will <strong>always</strong> throw an exception.
2504    * @param se the {@code ServiceException} instance to convert into an {@code IOException}
2505    */
2506   public static void toIOException(ServiceException se) throws IOException {
2507     if (se == null) {
2508       throw new NullPointerException("Null service exception passed!");
2509     }
2510
2511     Throwable cause = se.getCause();
2512     if (cause != null && cause instanceof IOException) {
2513       throw (IOException)cause;
2514     }
2515     throw new IOException(se);
2516   }
2517
2518   public static CellProtos.Cell toCell(final Cell kv) {
2519     // Doing this is going to kill us if we do it for all data passed.
2520     // St.Ack 20121205
2521     CellProtos.Cell.Builder kvbuilder = CellProtos.Cell.newBuilder();
2522     kvbuilder.setRow(ByteStringer.wrap(kv.getRowArray(), kv.getRowOffset(),
2523         kv.getRowLength()));
2524     kvbuilder.setFamily(ByteStringer.wrap(kv.getFamilyArray(),
2525         kv.getFamilyOffset(), kv.getFamilyLength()));
2526     kvbuilder.setQualifier(ByteStringer.wrap(kv.getQualifierArray(),
2527         kv.getQualifierOffset(), kv.getQualifierLength()));
2528     kvbuilder.setCellType(CellProtos.CellType.valueOf(kv.getTypeByte()));
2529     kvbuilder.setTimestamp(kv.getTimestamp());
2530     kvbuilder.setValue(ByteStringer.wrap(kv.getValueArray(), kv.getValueOffset(),
2531         kv.getValueLength()));
2532     return kvbuilder.build();
2533   }
2534
2535   public static Cell toCell(final CellProtos.Cell cell) {
2536     // Doing this is going to kill us if we do it for all data passed.
2537     // St.Ack 20121205
2538     return CellUtil.createCell(cell.getRow().toByteArray(),
2539       cell.getFamily().toByteArray(),
2540       cell.getQualifier().toByteArray(),
2541       cell.getTimestamp(),
2542       (byte)cell.getCellType().getNumber(),
2543       cell.getValue().toByteArray());
2544   }
2545
2546   public static HBaseProtos.NamespaceDescriptor toProtoNamespaceDescriptor(NamespaceDescriptor ns) {
2547     HBaseProtos.NamespaceDescriptor.Builder b =
2548         HBaseProtos.NamespaceDescriptor.newBuilder()
2549             .setName(ByteString.copyFromUtf8(ns.getName()));
2550     for(Map.Entry<String, String> entry: ns.getConfiguration().entrySet()) {
2551       b.addConfiguration(HBaseProtos.NameStringPair.newBuilder()
2552           .setName(entry.getKey())
2553           .setValue(entry.getValue()));
2554     }
2555     return b.build();
2556   }
2557
2558   public static NamespaceDescriptor toNamespaceDescriptor(
2559       HBaseProtos.NamespaceDescriptor desc) throws IOException {
2560     NamespaceDescriptor.Builder b =
2561       NamespaceDescriptor.create(desc.getName().toStringUtf8());
2562     for(HBaseProtos.NameStringPair prop : desc.getConfigurationList()) {
2563       b.addConfiguration(prop.getName(), prop.getValue());
2564     }
2565     return b.build();
2566   }
2567
2568   /**
2569    * Get an instance of the argument type declared in a class's signature. The
2570    * argument type is assumed to be a PB Message subclass, and the instance is
2571    * created using parseFrom method on the passed ByteString.
2572    * @param runtimeClass the runtime type of the class
2573    * @param position the position of the argument in the class declaration
2574    * @param b the ByteString which should be parsed to get the instance created
2575    * @return the instance
2576    * @throws IOException
2577    */
2578   @SuppressWarnings("unchecked")
2579   public static <T extends Message>
2580   T getParsedGenericInstance(Class<?> runtimeClass, int position, ByteString b)
2581       throws IOException {
2582     Type type = runtimeClass.getGenericSuperclass();
2583     Type argType = ((ParameterizedType)type).getActualTypeArguments()[position];
2584     Class<T> classType = (Class<T>)argType;
2585     T inst;
2586     try {
2587       Method m = classType.getMethod("parseFrom", ByteString.class);
2588       inst = (T)m.invoke(null, b);
2589       return inst;
2590     } catch (SecurityException e) {
2591       throw new IOException(e);
2592     } catch (NoSuchMethodException e) {
2593       throw new IOException(e);
2594     } catch (IllegalArgumentException e) {
2595       throw new IOException(e);
2596     } catch (InvocationTargetException e) {
2597       throw new IOException(e);
2598     } catch (IllegalAccessException e) {
2599       throw new IOException(e);
2600     }
2601   }
2602
2603   @SuppressWarnings("deprecation")
2604   public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family,
2605       List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
2606     return toCompactionDescriptor(info, null, family, inputPaths, outputPaths, storeDir);
2607   }
2608
2609   @SuppressWarnings("deprecation")
2610   public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] regionName,
2611       byte[] family, List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
2612     // compaction descriptor contains relative paths.
2613     // input / output paths are relative to the store dir
2614     // store dir is relative to region dir
2615     CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder()
2616         .setTableName(ByteStringer.wrap(info.getTable().toBytes()))
2617         .setEncodedRegionName(ByteStringer.wrap(
2618           regionName == null ? info.getEncodedNameAsBytes() : regionName))
2619         .setFamilyName(ByteStringer.wrap(family))
2620         .setStoreHomeDir(storeDir.getName()); //make relative
2621     for (Path inputPath : inputPaths) {
2622       builder.addCompactionInput(inputPath.getName()); //relative path
2623     }
2624     for (Path outputPath : outputPaths) {
2625       builder.addCompactionOutput(outputPath.getName());
2626     }
2627     builder.setRegionName(ByteStringer.wrap(info.getRegionName()));
2628     return builder.build();
2629   }
2630
2631   public static FlushDescriptor toFlushDescriptor(FlushAction action, HRegionInfo hri,
2632       long flushSeqId, Map<byte[], List<Path>> committedFiles) {
2633     FlushDescriptor.Builder desc = FlushDescriptor.newBuilder()
2634         .setAction(action)
2635         .setEncodedRegionName(ByteStringer.wrap(hri.getEncodedNameAsBytes()))
2636         .setRegionName(ByteStringer.wrap(hri.getRegionName()))
2637         .setFlushSequenceNumber(flushSeqId)
2638         .setTableName(ByteStringer.wrap(hri.getTable().getName()));
2639
2640     for (Map.Entry<byte[], List<Path>> entry : committedFiles.entrySet()) {
2641       WALProtos.FlushDescriptor.StoreFlushDescriptor.Builder builder =
2642           WALProtos.FlushDescriptor.StoreFlushDescriptor.newBuilder()
2643           .setFamilyName(ByteStringer.wrap(entry.getKey()))
2644           .setStoreHomeDir(Bytes.toString(entry.getKey())); //relative to region
2645       if (entry.getValue() != null) {
2646         for (Path path : entry.getValue()) {
2647           builder.addFlushOutput(path.getName());
2648         }
2649       }
2650       desc.addStoreFlushes(builder);
2651     }
2652     return desc.build();
2653   }
2654
2655   public static RegionEventDescriptor toRegionEventDescriptor(
2656       EventType eventType, HRegionInfo hri, long seqId, ServerName server,
2657       Map<byte[], List<Path>> storeFiles) {
2658     final byte[] tableNameAsBytes = hri.getTable().getName();
2659     final byte[] encodedNameAsBytes = hri.getEncodedNameAsBytes();
2660     final byte[] regionNameAsBytes = hri.getRegionName();
2661     return toRegionEventDescriptor(eventType,
2662         tableNameAsBytes,
2663         encodedNameAsBytes,
2664         regionNameAsBytes,
2665         seqId,
2666
2667         server,
2668         storeFiles);
2669   }
2670
2671   public static RegionEventDescriptor toRegionEventDescriptor(EventType eventType,
2672                                                               byte[] tableNameAsBytes,
2673                                                               byte[] encodedNameAsBytes,
2674                                                               byte[] regionNameAsBytes,
2675                                                                long seqId,
2676
2677                                                               ServerName server,
2678                                                               Map<byte[], List<Path>> storeFiles) {
2679     RegionEventDescriptor.Builder desc = RegionEventDescriptor.newBuilder()
2680         .setEventType(eventType)
2681         .setTableName(ByteStringer.wrap(tableNameAsBytes))
2682         .setEncodedRegionName(ByteStringer.wrap(encodedNameAsBytes))
2683         .setRegionName(ByteStringer.wrap(regionNameAsBytes))
2684         .setLogSequenceNumber(seqId)
2685         .setServer(toServerName(server));
2686
2687     for (Entry<byte[], List<Path>> entry : storeFiles.entrySet()) {
2688       StoreDescriptor.Builder builder = StoreDescriptor.newBuilder()
2689           .setFamilyName(ByteStringer.wrap(entry.getKey()))
2690           .setStoreHomeDir(Bytes.toString(entry.getKey()));
2691       for (Path path : entry.getValue()) {
2692         builder.addStoreFile(path.getName());
2693       }
2694
2695       desc.addStores(builder);
2696     }
2697     return desc.build();
2698   }
2699
2700   /**
2701    * Return short version of Message toString'd, shorter than TextFormat#shortDebugString.
2702    * Tries to NOT print out data both because it can be big but also so we do not have data in our
2703    * logs. Use judiciously.
2704    * @param m
2705    * @return toString of passed <code>m</code>
2706    */
2707   public static String getShortTextFormat(Message m) {
2708     if (m == null) return "null";
2709     if (m instanceof ScanRequest) {
2710       // This should be small and safe to output.  No data.
2711       return TextFormat.shortDebugString(m);
2712     } else if (m instanceof RegionServerReportRequest) {
2713       // Print a short message only, just the servername and the requests, not the full load.
2714       RegionServerReportRequest r = (RegionServerReportRequest)m;
2715       return "server " + TextFormat.shortDebugString(r.getServer()) +
2716         " load { numberOfRequests: " + r.getLoad().getNumberOfRequests() + " }";
2717     } else if (m instanceof RegionServerStartupRequest) {
2718       // Should be small enough.
2719       return TextFormat.shortDebugString(m);
2720     } else if (m instanceof MutationProto) {
2721       return toShortString((MutationProto)m);
2722     } else if (m instanceof GetRequest) {
2723       GetRequest r = (GetRequest) m;
2724       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2725           ", row=" + getStringForByteString(r.getGet().getRow());
2726     } else if (m instanceof ClientProtos.MultiRequest) {
2727       ClientProtos.MultiRequest r = (ClientProtos.MultiRequest) m;
2728       // Get first set of Actions.
2729       ClientProtos.RegionAction actions = r.getRegionActionList().get(0);
2730       String row = actions.getActionCount() <= 0? "":
2731         getStringForByteString(actions.getAction(0).hasGet()?
2732           actions.getAction(0).getGet().getRow():
2733           actions.getAction(0).getMutation().getRow());
2734       return "region= " + getStringForByteString(actions.getRegion().getValue()) +
2735           ", for " + r.getRegionActionCount() +
2736           " actions and 1st row key=" + row;
2737     } else if (m instanceof ClientProtos.MutateRequest) {
2738       ClientProtos.MutateRequest r = (ClientProtos.MutateRequest) m;
2739       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2740           ", row=" + getStringForByteString(r.getMutation().getRow());
2741     }
2742     return "TODO: " + m.getClass().toString();
2743   }
2744
2745   private static String getStringForByteString(ByteString bs) {
2746     return Bytes.toStringBinary(bs.toByteArray());
2747   }
2748
2749   /**
2750    * Print out some subset of a MutationProto rather than all of it and its data
2751    * @param proto Protobuf to print out
2752    * @return Short String of mutation proto
2753    */
2754   static String toShortString(final MutationProto proto) {
2755     return "row=" + Bytes.toString(proto.getRow().toByteArray()) +
2756         ", type=" + proto.getMutateType().toString();
2757   }
2758
2759   public static TableName toTableName(HBaseProtos.TableName tableNamePB) {
2760     return TableName.valueOf(tableNamePB.getNamespace().asReadOnlyByteBuffer(),
2761         tableNamePB.getQualifier().asReadOnlyByteBuffer());
2762   }
2763
2764   public static HBaseProtos.TableName toProtoTableName(TableName tableName) {
2765     return HBaseProtos.TableName.newBuilder()
2766         .setNamespace(ByteStringer.wrap(tableName.getNamespace()))
2767         .setQualifier(ByteStringer.wrap(tableName.getQualifier())).build();
2768   }
2769
2770   public static TableName[] getTableNameArray(List<HBaseProtos.TableName> tableNamesList) {
2771     if (tableNamesList == null) {
2772       return new TableName[0];
2773     }
2774     TableName[] tableNames = new TableName[tableNamesList.size()];
2775     for (int i = 0; i < tableNamesList.size(); i++) {
2776       tableNames[i] = toTableName(tableNamesList.get(i));
2777     }
2778     return tableNames;
2779   }
2780
2781   /**
2782    * Convert a protocol buffer CellVisibility to a client CellVisibility
2783    *
2784    * @param proto
2785    * @return the converted client CellVisibility
2786    */
2787   public static CellVisibility toCellVisibility(ClientProtos.CellVisibility proto) {
2788     if (proto == null) return null;
2789     return new CellVisibility(proto.getExpression());
2790   }
2791
2792   /**
2793    * Convert a protocol buffer CellVisibility bytes to a client CellVisibility
2794    *
2795    * @param protoBytes
2796    * @return the converted client CellVisibility
2797    * @throws DeserializationException
2798    */
2799   public static CellVisibility toCellVisibility(byte[] protoBytes) throws DeserializationException {
2800     if (protoBytes == null) return null;
2801     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2802     ClientProtos.CellVisibility proto = null;
2803     try {
2804       ProtobufUtil.mergeFrom(builder, protoBytes);
2805       proto = builder.build();
2806     } catch (IOException e) {
2807       throw new DeserializationException(e);
2808     }
2809     return toCellVisibility(proto);
2810   }
2811
2812   /**
2813    * Create a protocol buffer CellVisibility based on a client CellVisibility.
2814    *
2815    * @param cellVisibility
2816    * @return a protocol buffer CellVisibility
2817    */
2818   public static ClientProtos.CellVisibility toCellVisibility(CellVisibility cellVisibility) {
2819     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2820     builder.setExpression(cellVisibility.getExpression());
2821     return builder.build();
2822   }
2823
2824   /**
2825    * Convert a protocol buffer Authorizations to a client Authorizations
2826    *
2827    * @param proto
2828    * @return the converted client Authorizations
2829    */
2830   public static Authorizations toAuthorizations(ClientProtos.Authorizations proto) {
2831     if (proto == null) return null;
2832     return new Authorizations(proto.getLabelList());
2833   }
2834
2835   /**
2836    * Convert a protocol buffer Authorizations bytes to a client Authorizations
2837    *
2838    * @param protoBytes
2839    * @return the converted client Authorizations
2840    * @throws DeserializationException
2841    */
2842   public static Authorizations toAuthorizations(byte[] protoBytes) throws DeserializationException {
2843     if (protoBytes == null) return null;
2844     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2845     ClientProtos.Authorizations proto = null;
2846     try {
2847       ProtobufUtil.mergeFrom(builder, protoBytes);
2848       proto = builder.build();
2849     } catch (IOException e) {
2850       throw new DeserializationException(e);
2851     }
2852     return toAuthorizations(proto);
2853   }
2854
2855   /**
2856    * Create a protocol buffer Authorizations based on a client Authorizations.
2857    *
2858    * @param authorizations
2859    * @return a protocol buffer Authorizations
2860    */
2861   public static ClientProtos.Authorizations toAuthorizations(Authorizations authorizations) {
2862     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2863     for (String label : authorizations.getLabels()) {
2864       builder.addLabel(label);
2865     }
2866     return builder.build();
2867   }
2868
2869   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(String user,
2870       Permission perms) {
2871     return AccessControlProtos.UsersAndPermissions.newBuilder()
2872       .addUserPermissions(AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder()
2873         .setUser(ByteString.copyFromUtf8(user))
2874         .addPermissions(toPermission(perms))
2875         .build())
2876       .build();
2877   }
2878
2879   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(
2880       ListMultimap<String, Permission> perms) {
2881     AccessControlProtos.UsersAndPermissions.Builder builder =
2882         AccessControlProtos.UsersAndPermissions.newBuilder();
2883     for (Map.Entry<String, Collection<Permission>> entry : perms.asMap().entrySet()) {
2884       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
2885         AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
2886       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
2887       for (Permission perm: entry.getValue()) {
2888         userPermBuilder.addPermissions(toPermission(perm));
2889       }
2890       builder.addUserPermissions(userPermBuilder.build());
2891     }
2892     return builder.build();
2893   }
2894
2895   public static ListMultimap<String, Permission> toUsersAndPermissions(
2896       AccessControlProtos.UsersAndPermissions proto) {
2897     ListMultimap<String, Permission> result = ArrayListMultimap.create();
2898     for (AccessControlProtos.UsersAndPermissions.UserPermissions userPerms:
2899         proto.getUserPermissionsList()) {
2900       String user = userPerms.getUser().toStringUtf8();
2901       for (AccessControlProtos.Permission perm: userPerms.getPermissionsList()) {
2902         result.put(user, toPermission(perm));
2903       }
2904     }
2905     return result;
2906   }
2907
2908   /**
2909    * Convert a protocol buffer TimeUnit to a client TimeUnit
2910    *
2911    * @param proto
2912    * @return the converted client TimeUnit
2913    */
2914   public static TimeUnit toTimeUnit(final HBaseProtos.TimeUnit proto) {
2915     switch (proto) {
2916       case NANOSECONDS:  return TimeUnit.NANOSECONDS;
2917       case MICROSECONDS: return TimeUnit.MICROSECONDS;
2918       case MILLISECONDS: return TimeUnit.MILLISECONDS;
2919       case SECONDS:      return TimeUnit.SECONDS;
2920       case MINUTES:      return TimeUnit.MINUTES;
2921       case HOURS:        return TimeUnit.HOURS;
2922       case DAYS:         return TimeUnit.DAYS;
2923     }
2924     throw new RuntimeException("Invalid TimeUnit " + proto);
2925   }
2926
2927   /**
2928    * Convert a client TimeUnit to a protocol buffer TimeUnit
2929    *
2930    * @param timeUnit
2931    * @return the converted protocol buffer TimeUnit
2932    */
2933   public static HBaseProtos.TimeUnit toProtoTimeUnit(final TimeUnit timeUnit) {
2934     switch (timeUnit) {
2935       case NANOSECONDS:  return HBaseProtos.TimeUnit.NANOSECONDS;
2936       case MICROSECONDS: return HBaseProtos.TimeUnit.MICROSECONDS;
2937       case MILLISECONDS: return HBaseProtos.TimeUnit.MILLISECONDS;
2938       case SECONDS:      return HBaseProtos.TimeUnit.SECONDS;
2939       case MINUTES:      return HBaseProtos.TimeUnit.MINUTES;
2940       case HOURS:        return HBaseProtos.TimeUnit.HOURS;
2941       case DAYS:         return HBaseProtos.TimeUnit.DAYS;
2942     }
2943     throw new RuntimeException("Invalid TimeUnit " + timeUnit);
2944   }
2945
2946   /**
2947    * Convert a protocol buffer ThrottleType to a client ThrottleType
2948    *
2949    * @param proto
2950    * @return the converted client ThrottleType
2951    */
2952   public static ThrottleType toThrottleType(final QuotaProtos.ThrottleType proto) {
2953     switch (proto) {
2954       case REQUEST_NUMBER: return ThrottleType.REQUEST_NUMBER;
2955       case REQUEST_SIZE:   return ThrottleType.REQUEST_SIZE;
2956       case WRITE_NUMBER:   return ThrottleType.WRITE_NUMBER;
2957       case WRITE_SIZE:     return ThrottleType.WRITE_SIZE;
2958       case READ_NUMBER:    return ThrottleType.READ_NUMBER;
2959       case READ_SIZE:      return ThrottleType.READ_SIZE;
2960     }
2961     throw new RuntimeException("Invalid ThrottleType " + proto);
2962   }
2963
2964   /**
2965    * Convert a client ThrottleType to a protocol buffer ThrottleType
2966    *
2967    * @param type
2968    * @return the converted protocol buffer ThrottleType
2969    */
2970   public static QuotaProtos.ThrottleType toProtoThrottleType(final ThrottleType type) {
2971     switch (type) {
2972       case REQUEST_NUMBER: return QuotaProtos.ThrottleType.REQUEST_NUMBER;
2973       case REQUEST_SIZE:   return QuotaProtos.ThrottleType.REQUEST_SIZE;
2974       case WRITE_NUMBER:   return QuotaProtos.ThrottleType.WRITE_NUMBER;
2975       case WRITE_SIZE:     return QuotaProtos.ThrottleType.WRITE_SIZE;
2976       case READ_NUMBER:    return QuotaProtos.ThrottleType.READ_NUMBER;
2977       case READ_SIZE:      return QuotaProtos.ThrottleType.READ_SIZE;
2978     }
2979     throw new RuntimeException("Invalid ThrottleType " + type);
2980   }
2981
2982   /**
2983    * Convert a protocol buffer QuotaScope to a client QuotaScope
2984    *
2985    * @param proto
2986    * @return the converted client QuotaScope
2987    */
2988   public static QuotaScope toQuotaScope(final QuotaProtos.QuotaScope proto) {
2989     switch (proto) {
2990       case CLUSTER: return QuotaScope.CLUSTER;
2991       case MACHINE: return QuotaScope.MACHINE;
2992     }
2993     throw new RuntimeException("Invalid QuotaScope " + proto);
2994   }
2995
2996   /**
2997    * Convert a client QuotaScope to a protocol buffer QuotaScope
2998    *
2999    * @param scope
3000    * @return the converted protocol buffer QuotaScope
3001    */
3002   public static QuotaProtos.QuotaScope toProtoQuotaScope(final QuotaScope scope) {
3003     switch (scope) {
3004       case CLUSTER: return QuotaProtos.QuotaScope.CLUSTER;
3005       case MACHINE: return QuotaProtos.QuotaScope.MACHINE;
3006     }
3007     throw new RuntimeException("Invalid QuotaScope " + scope);
3008   }
3009
3010   /**
3011    * Convert a protocol buffer QuotaType to a client QuotaType
3012    *
3013    * @param proto
3014    * @return the converted client QuotaType
3015    */
3016   public static QuotaType toQuotaScope(final QuotaProtos.QuotaType proto) {
3017     switch (proto) {
3018       case THROTTLE: return QuotaType.THROTTLE;
3019     }
3020     throw new RuntimeException("Invalid QuotaType " + proto);
3021   }
3022
3023   /**
3024    * Convert a client QuotaType to a protocol buffer QuotaType
3025    *
3026    * @param type
3027    * @return the converted protocol buffer QuotaType
3028    */
3029   public static QuotaProtos.QuotaType toProtoQuotaScope(final QuotaType type) {
3030     switch (type) {
3031       case THROTTLE: return QuotaProtos.QuotaType.THROTTLE;
3032     }
3033     throw new RuntimeException("Invalid QuotaType " + type);
3034   }
3035
3036   /**
3037    * Build a protocol buffer TimedQuota
3038    *
3039    * @param limit the allowed number of request/data per timeUnit
3040    * @param timeUnit the limit time unit
3041    * @param scope the quota scope
3042    * @return the protocol buffer TimedQuota
3043    */
3044   public static QuotaProtos.TimedQuota toTimedQuota(final long limit, final TimeUnit timeUnit,
3045       final QuotaScope scope) {
3046     return QuotaProtos.TimedQuota.newBuilder()
3047             .setSoftLimit(limit)
3048             .setTimeUnit(toProtoTimeUnit(timeUnit))
3049             .setScope(toProtoQuotaScope(scope))
3050             .build();
3051   }
3052
3053   /**
3054    * Generates a marker for the WAL so that we propagate the notion of a bulk region load
3055    * throughout the WAL.
3056    *
3057    * @param tableName         The tableName into which the bulk load is being imported into.
3058    * @param encodedRegionName Encoded region name of the region which is being bulk loaded.
3059    * @param storeFiles        A set of store files of a column family are bulk loaded.
3060    * @param storeFilesSize  Map of store files and their lengths
3061    * @param bulkloadSeqId     sequence ID (by a force flush) used to create bulk load hfile
3062    *                          name
3063    * @return The WAL log marker for bulk loads.
3064    */
3065   public static WALProtos.BulkLoadDescriptor toBulkLoadDescriptor(TableName tableName,
3066       ByteString encodedRegionName, Map<byte[], List<Path>> storeFiles,
3067       Map<String, Long> storeFilesSize, long bulkloadSeqId) {
3068     BulkLoadDescriptor.Builder desc =
3069         BulkLoadDescriptor.newBuilder()
3070         .setTableName(ProtobufUtil.toProtoTableName(tableName))
3071         .setEncodedRegionName(encodedRegionName).setBulkloadSeqNum(bulkloadSeqId);
3072
3073     for (Map.Entry<byte[], List<Path>> entry : storeFiles.entrySet()) {
3074       WALProtos.StoreDescriptor.Builder builder = StoreDescriptor.newBuilder()
3075           .setFamilyName(ByteStringer.wrap(entry.getKey()))
3076           .setStoreHomeDir(Bytes.toString(entry.getKey())); // relative to region
3077       for (Path path : entry.getValue()) {
3078         String name = path.getName();
3079         builder.addStoreFile(name);
3080         Long size = storeFilesSize.get(name) == null ? (Long) 0L : storeFilesSize.get(name);
3081         builder.setStoreFileSizeBytes(size);
3082       }
3083       desc.addStores(builder);
3084     }
3085
3086     return desc.build();
3087   }
3088
3089   /**
3090    * This version of protobuf's mergeDelimitedFrom avoid the hard-coded 64MB limit for decoding
3091    * buffers
3092    * @param builder current message builder
3093    * @param in Inputsream with delimited protobuf data
3094    * @throws IOException
3095    */
3096   public static void mergeDelimitedFrom(Message.Builder builder, InputStream in)
3097     throws IOException {
3098     // This used to be builder.mergeDelimitedFrom(in);
3099     // but is replaced to allow us to bump the protobuf size limit.
3100     final int firstByte = in.read();
3101     if (firstByte != -1) {
3102       final int size = CodedInputStream.readRawVarint32(firstByte, in);
3103       final InputStream limitedInput = new LimitInputStream(in, size);
3104       final CodedInputStream codedInput = CodedInputStream.newInstance(limitedInput);
3105       codedInput.setSizeLimit(size);
3106       builder.mergeFrom(codedInput);
3107       codedInput.checkLastTagWas(0);
3108     }
3109   }
3110
3111   /**
3112    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3113    * buffers where the message size is known
3114    * @param builder current message builder
3115    * @param in InputStream containing protobuf data
3116    * @param size known size of protobuf data
3117    * @throws IOException
3118    */
3119   public static void mergeFrom(Message.Builder builder, InputStream in, int size)
3120       throws IOException {
3121     final CodedInputStream codedInput = CodedInputStream.newInstance(in);
3122     codedInput.setSizeLimit(size);
3123     builder.mergeFrom(codedInput);
3124     codedInput.checkLastTagWas(0);
3125   }
3126
3127   /**
3128    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3129    * buffers where the message size is not known
3130    * @param builder current message builder
3131    * @param in InputStream containing protobuf data
3132    * @throws IOException
3133    */
3134   public static void mergeFrom(Message.Builder builder, InputStream in)
3135       throws IOException {
3136     final CodedInputStream codedInput = CodedInputStream.newInstance(in);
3137     codedInput.setSizeLimit(Integer.MAX_VALUE);
3138     builder.mergeFrom(codedInput);
3139     codedInput.checkLastTagWas(0);
3140   }
3141
3142   /**
3143    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3144    * buffers when working with ByteStrings
3145    * @param builder current message builder
3146    * @param bs ByteString containing the
3147    * @throws IOException
3148    */
3149   public static void mergeFrom(Message.Builder builder, ByteString bs) throws IOException {
3150     final CodedInputStream codedInput = bs.newCodedInput();
3151     codedInput.setSizeLimit(bs.size());
3152     builder.mergeFrom(codedInput);
3153     codedInput.checkLastTagWas(0);
3154   }
3155
3156   /**
3157    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3158    * buffers when working with byte arrays
3159    * @param builder current message builder
3160    * @param b byte array
3161    * @throws IOException
3162    */
3163   public static void mergeFrom(Message.Builder builder, byte[] b) throws IOException {
3164     final CodedInputStream codedInput = CodedInputStream.newInstance(b);
3165     codedInput.setSizeLimit(b.length);
3166     builder.mergeFrom(codedInput);
3167     codedInput.checkLastTagWas(0);
3168   }
3169
3170   /**
3171    * This version of protobuf's mergeFrom avoids the hard-coded 64MB limit for decoding
3172    * buffers when working with byte arrays
3173    * @param builder current message builder
3174    * @param b byte array
3175    * @param offset
3176    * @param length
3177    * @throws IOException
3178    */
3179   public static void mergeFrom(Message.Builder builder, byte[] b, int offset, int length)
3180       throws IOException {
3181     final CodedInputStream codedInput = CodedInputStream.newInstance(b, offset, length);
3182     codedInput.setSizeLimit(length);
3183     builder.mergeFrom(codedInput);
3184     codedInput.checkLastTagWas(0);
3185   }
3186
3187   public static void mergeFrom(Message.Builder builder, CodedInputStream codedInput, int length)
3188       throws IOException {
3189     codedInput.resetSizeCounter();
3190     int prevLimit = codedInput.setSizeLimit(length);
3191
3192     int limit = codedInput.pushLimit(length);
3193     builder.mergeFrom(codedInput);
3194     codedInput.popLimit(limit);
3195
3196     codedInput.checkLastTagWas(0);
3197     codedInput.setSizeLimit(prevLimit);
3198   }
3199
3200   public static ReplicationLoadSink toReplicationLoadSink(
3201       ClusterStatusProtos.ReplicationLoadSink cls) {
3202     return new ReplicationLoadSink(cls.getAgeOfLastAppliedOp(), cls.getTimeStampsOfLastAppliedOp());
3203   }
3204
3205   public static ReplicationLoadSource toReplicationLoadSource(
3206       ClusterStatusProtos.ReplicationLoadSource cls) {
3207     return new ReplicationLoadSource(cls.getPeerID(), cls.getAgeOfLastShippedOp(),
3208         cls.getSizeOfLogQueue(), cls.getTimeStampOfLastShippedOp(), cls.getReplicationLag());
3209   }
3210
3211   public static List<ReplicationLoadSource> toReplicationLoadSourceList(
3212       List<ClusterStatusProtos.ReplicationLoadSource> clsList) {
3213     ArrayList<ReplicationLoadSource> rlsList = new ArrayList<ReplicationLoadSource>();
3214     for (ClusterStatusProtos.ReplicationLoadSource cls : clsList) {
3215       rlsList.add(toReplicationLoadSource(cls));
3216     }
3217     return rlsList;
3218   }
3219
3220   /**
3221    * Get a protocol buffer VersionInfo
3222    *
3223    * @return the converted protocol buffer VersionInfo
3224    */
3225   public static HBaseProtos.VersionInfo getVersionInfo() {
3226     HBaseProtos.VersionInfo.Builder builder = HBaseProtos.VersionInfo.newBuilder();
3227     String version = VersionInfo.getVersion();
3228     builder.setVersion(version);
3229     String[] components = version.split("\\.");
3230     if (components != null && components.length > 2) {
3231       builder.setVersionMajor(Integer.parseInt(components[0]));
3232       builder.setVersionMinor(Integer.parseInt(components[1]));
3233     }
3234     builder.setUrl(VersionInfo.getUrl());
3235     builder.setRevision(VersionInfo.getRevision());
3236     builder.setUser(VersionInfo.getUser());
3237     builder.setDate(VersionInfo.getDate());
3238     builder.setSrcChecksum(VersionInfo.getSrcChecksum());
3239     return builder.build();
3240   }
3241
3242   /**
3243    * Convert SecurityCapabilitiesResponse.Capability to SecurityCapability
3244    * @param capabilities capabilities returned in the SecurityCapabilitiesResponse message
3245    * @return the converted list of SecurityCapability elements
3246    */
3247   public static List<SecurityCapability> toSecurityCapabilityList(
3248       List<MasterProtos.SecurityCapabilitiesResponse.Capability> capabilities) {
3249     List<SecurityCapability> scList = new ArrayList<>(capabilities.size());
3250     for (MasterProtos.SecurityCapabilitiesResponse.Capability c: capabilities) {
3251       try {
3252         scList.add(SecurityCapability.valueOf(c.getNumber()));
3253       } catch (IllegalArgumentException e) {
3254         // Unknown capability, just ignore it. We don't understand the new capability
3255         // but don't care since by definition we cannot take advantage of it.
3256       }
3257     }
3258     return scList;
3259   }
3260
3261   private static HBaseProtos.TimeRange.Builder timeRangeToProto(TimeRange timeRange) {
3262     HBaseProtos.TimeRange.Builder timeRangeBuilder =
3263         HBaseProtos.TimeRange.newBuilder();
3264     timeRangeBuilder.setFrom(timeRange.getMin());
3265     timeRangeBuilder.setTo(timeRange.getMax());
3266     return timeRangeBuilder;
3267   }
3268
3269   private static TimeRange protoToTimeRange(HBaseProtos.TimeRange timeRange) throws IOException {
3270       long minStamp = 0;
3271       long maxStamp = Long.MAX_VALUE;
3272       if (timeRange.hasFrom()) {
3273         minStamp = timeRange.getFrom();
3274       }
3275       if (timeRange.hasTo()) {
3276         maxStamp = timeRange.getTo();
3277       }
3278     return new TimeRange(minStamp, maxStamp);
3279   }
3280
3281   public static RSGroupInfo toGroupInfo(RSGroupProtos.RSGroupInfo proto) {
3282     RSGroupInfo RSGroupInfo = new RSGroupInfo(proto.getName());
3283     for(HBaseProtos.ServerName el: proto.getServersList()) {
3284       RSGroupInfo.addServer(HostAndPort.fromParts(el.getHostName(), el.getPort()));
3285     }
3286     for(HBaseProtos.TableName pTableName: proto.getTablesList()) {
3287       RSGroupInfo.addTable(ProtobufUtil.toTableName(pTableName));
3288     }
3289     return RSGroupInfo;
3290   }
3291
3292   public static RSGroupProtos.RSGroupInfo toProtoGroupInfo(RSGroupInfo pojo) {
3293     List<HBaseProtos.TableName> tables =
3294         new ArrayList<HBaseProtos.TableName>(pojo.getTables().size());
3295     for(TableName arg: pojo.getTables()) {
3296       tables.add(ProtobufUtil.toProtoTableName(arg));
3297     }
3298     List<HBaseProtos.ServerName> hostports =
3299         new ArrayList<HBaseProtos.ServerName>(pojo.getServers().size());
3300     for(HostAndPort el: pojo.getServers()) {
3301       hostports.add(HBaseProtos.ServerName.newBuilder()
3302           .setHostName(el.getHostText())
3303           .setPort(el.getPort())
3304           .build());
3305     }
3306     return RSGroupProtos.RSGroupInfo.newBuilder().setName(pojo.getName())
3307         .addAllServers(hostports)
3308         .addAllTables(tables).build();
3309   }
3310
3311   /**
3312    * Converts an HColumnDescriptor to ColumnFamilySchema
3313    * @param hcd the HColummnDescriptor
3314    * @return Convert this instance to a the pb column family type
3315    */
3316   public static ColumnFamilySchema convertToColumnFamilySchema(HColumnDescriptor hcd) {
3317     ColumnFamilySchema.Builder builder = ColumnFamilySchema.newBuilder();
3318     builder.setName(ByteStringer.wrap(hcd.getName()));
3319     for (Map.Entry<Bytes, Bytes> e : hcd.getValues().entrySet()) {
3320       BytesBytesPair.Builder aBuilder = BytesBytesPair.newBuilder();
3321       aBuilder.setFirst(ByteStringer.wrap(e.getKey().get()));
3322       aBuilder.setSecond(ByteStringer.wrap(e.getValue().get()));
3323       builder.addAttributes(aBuilder.build());
3324     }
3325     for (Map.Entry<String, String> e : hcd.getConfiguration().entrySet()) {
3326       NameStringPair.Builder aBuilder = NameStringPair.newBuilder();
3327       aBuilder.setName(e.getKey());
3328       aBuilder.setValue(e.getValue());
3329       builder.addConfiguration(aBuilder.build());
3330     }
3331     return builder.build();
3332   }
3333
3334   /**
3335    * Converts a ColumnFamilySchema to HColumnDescriptor
3336    * @param cfs the ColumnFamilySchema
3337    * @return An {@link HColumnDescriptor} made from the passed in <code>cfs</code>
3338    */
3339   public static HColumnDescriptor convertToHColumnDesc(final ColumnFamilySchema cfs) {
3340     // Use the empty constructor so we preserve the initial values set on construction for things
3341     // like maxVersion.  Otherwise, we pick up wrong values on deserialization which makes for
3342     // unrelated-looking test failures that are hard to trace back to here.
3343     HColumnDescriptor hcd = new HColumnDescriptor(cfs.getName().toByteArray());
3344     for (BytesBytesPair a: cfs.getAttributesList()) {
3345       hcd.setValue(a.getFirst().toByteArray(), a.getSecond().toByteArray());
3346     }
3347     for (NameStringPair a: cfs.getConfigurationList()) {
3348       hcd.setConfiguration(a.getName(), a.getValue());
3349     }
3350     return hcd;
3351   }
3352
3353   /**
3354    * Converts an HTableDescriptor to TableSchema
3355    * @param htd the HTableDescriptor
3356    * @return Convert the current {@link HTableDescriptor} into a pb TableSchema instance.
3357    */
3358   public static TableSchema convertToTableSchema(HTableDescriptor htd) {
3359     TableSchema.Builder builder = TableSchema.newBuilder();
3360     builder.setTableName(toProtoTableName(htd.getTableName()));
3361     for (Map.Entry<Bytes, Bytes> e : htd.getValues().entrySet()) {
3362       BytesBytesPair.Builder aBuilder = BytesBytesPair.newBuilder();
3363       aBuilder.setFirst(ByteStringer.wrap(e.getKey().get()));
3364       aBuilder.setSecond(ByteStringer.wrap(e.getValue().get()));
3365       builder.addAttributes(aBuilder.build());
3366     }
3367     for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
3368       builder.addColumnFamilies(convertToColumnFamilySchema(hcd));
3369     }
3370     for (Map.Entry<String, String> e : htd.getConfiguration().entrySet()) {
3371       NameStringPair.Builder aBuilder = NameStringPair.newBuilder();
3372       aBuilder.setName(e.getKey());
3373       aBuilder.setValue(e.getValue());
3374       builder.addConfiguration(aBuilder.build());
3375     }
3376     return builder.build();
3377   }
3378
3379   /**
3380    * Converts a TableSchema to HTableDescriptor
3381    * @param ts A pb TableSchema instance.
3382    * @return An {@link HTableDescriptor} made from the passed in pb <code>ts</code>.
3383    */
3384   public static HTableDescriptor convertToHTableDesc(final TableSchema ts) {
3385     List<ColumnFamilySchema> list = ts.getColumnFamiliesList();
3386     HColumnDescriptor [] hcds = new HColumnDescriptor[list.size()];
3387     int index = 0;
3388     for (ColumnFamilySchema cfs: list) {
3389       hcds[index++] = ProtobufUtil.convertToHColumnDesc(cfs);
3390     }
3391     HTableDescriptor htd = new HTableDescriptor(ProtobufUtil.toTableName(ts.getTableName()));
3392     for (HColumnDescriptor hcd : hcds) {
3393       htd.addFamily(hcd);
3394     }
3395     for (BytesBytesPair a: ts.getAttributesList()) {
3396       htd.setValue(a.getFirst().toByteArray(), a.getSecond().toByteArray());
3397     }
3398     for (NameStringPair a: ts.getConfigurationList()) {
3399       htd.setConfiguration(a.getName(), a.getValue());
3400     }
3401     return htd;
3402   }
3403
3404   /**
3405    * Creates {@link CompactionState} from
3406    * {@link org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState}
3407    * state
3408    * @param state the protobuf CompactionState
3409    * @return CompactionState
3410    */
3411   public static CompactionState createCompactionState(GetRegionInfoResponse.CompactionState state) {
3412     return CompactionState.valueOf(state.toString());
3413   }
3414
3415   /**
3416    * Creates {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type}
3417    * from {@link SnapshotType}
3418    * @param type the SnapshotDescription type
3419    * @return the protobuf SnapshotDescription type
3420    */
3421   public static HBaseProtos.SnapshotDescription.Type
3422       createProtosSnapShotDescType(SnapshotType type) {
3423     return HBaseProtos.SnapshotDescription.Type.valueOf(type.name());
3424   }
3425
3426   /**
3427    * Creates {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type}
3428    * from the type of SnapshotDescription string
3429    * @param snapshotDesc string representing the snapshot description type
3430    * @return the protobuf SnapshotDescription type
3431    */
3432   public static HBaseProtos.SnapshotDescription.Type
3433       createProtosSnapShotDescType(String snapshotDesc) {
3434     return HBaseProtos.SnapshotDescription.Type.valueOf(snapshotDesc.toUpperCase(Locale.ROOT));
3435   }
3436
3437   /**
3438    * Creates {@link SnapshotType} from the type of
3439    * {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription}
3440    * @param type the snapshot description type
3441    * @return the protobuf SnapshotDescription type
3442    */
3443   public static SnapshotType createSnapshotType(HBaseProtos.SnapshotDescription.Type type) {
3444     return SnapshotType.valueOf(type.toString());
3445   }
3446
3447   /**
3448    * Convert from {@link SnapshotDescription} to
3449    * {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription}
3450    * @param snapshotDesc the POJO SnapshotDescription
3451    * @return the protobuf SnapshotDescription
3452    */
3453   public static HBaseProtos.SnapshotDescription
3454       createHBaseProtosSnapshotDesc(SnapshotDescription snapshotDesc) {
3455     HBaseProtos.SnapshotDescription.Builder builder = HBaseProtos.SnapshotDescription.newBuilder();
3456     if (snapshotDesc.getTable() != null) {
3457       builder.setTable(snapshotDesc.getTable());
3458     }
3459     if (snapshotDesc.getName() != null) {
3460       builder.setName(snapshotDesc.getName());
3461     }
3462     if (snapshotDesc.getOwner() != null) {
3463       builder.setOwner(snapshotDesc.getOwner());
3464     }
3465     if (snapshotDesc.getCreationTime() != -1L) {
3466       builder.setCreationTime(snapshotDesc.getCreationTime());
3467     }
3468     if (snapshotDesc.getVersion() != -1) {
3469       builder.setVersion(snapshotDesc.getVersion());
3470     }
3471     builder.setType(ProtobufUtil.createProtosSnapShotDescType(snapshotDesc.getType()));
3472     HBaseProtos.SnapshotDescription snapshot = builder.build();
3473     return snapshot;
3474   }
3475
3476   /**
3477    * Convert from
3478    * {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} to
3479    * {@link SnapshotDescription}
3480    * @param snapshotDesc the protobuf SnapshotDescription
3481    * @return the POJO SnapshotDescription
3482    */
3483   public static SnapshotDescription
3484       createSnapshotDesc(HBaseProtos.SnapshotDescription snapshotDesc) {
3485     return new SnapshotDescription(snapshotDesc.getName(), snapshotDesc.getTable(),
3486         createSnapshotType(snapshotDesc.getType()), snapshotDesc.getOwner(),
3487         snapshotDesc.getCreationTime(), snapshotDesc.getVersion());
3488   }
3489
3490   /**
3491    * Convert a protobuf ClusterStatus to a ClusterStatus
3492    *
3493    * @param proto the protobuf ClusterStatus
3494    * @return the converted ClusterStatus
3495    */
3496   public static ClusterStatus convert(ClusterStatusProtos.ClusterStatus proto) {
3497
3498     Map<ServerName, ServerLoad> servers = null;
3499     servers = new HashMap<ServerName, ServerLoad>(proto.getLiveServersList().size());
3500     for (LiveServerInfo lsi : proto.getLiveServersList()) {
3501       servers.put(ProtobufUtil.toServerName(
3502           lsi.getServer()), new ServerLoad(lsi.getServerLoad()));
3503     }
3504
3505     Collection<ServerName> deadServers = null;
3506     deadServers = new ArrayList<ServerName>(proto.getDeadServersList().size());
3507     for (HBaseProtos.ServerName sn : proto.getDeadServersList()) {
3508       deadServers.add(ProtobufUtil.toServerName(sn));
3509     }
3510
3511     Collection<ServerName> backupMasters = null;
3512     backupMasters = new ArrayList<ServerName>(proto.getBackupMastersList().size());
3513     for (HBaseProtos.ServerName sn : proto.getBackupMastersList()) {
3514       backupMasters.add(ProtobufUtil.toServerName(sn));
3515     }
3516
3517     Set<RegionState> rit = null;
3518     rit = new HashSet<RegionState>(proto.getRegionsInTransitionList().size());
3519     for (RegionInTransition region : proto.getRegionsInTransitionList()) {
3520       RegionState value = RegionState.convert(region.getRegionState());
3521       rit.add(value);
3522     }
3523
3524     String[] masterCoprocessors = null;
3525     final int numMasterCoprocessors = proto.getMasterCoprocessorsCount();
3526     masterCoprocessors = new String[numMasterCoprocessors];
3527     for (int i = 0; i < numMasterCoprocessors; i++) {
3528       masterCoprocessors[i] = proto.getMasterCoprocessors(i).getName();
3529     }
3530
3531     return new ClusterStatus(proto.getHbaseVersion().getVersion(),
3532       ClusterId.convert(proto.getClusterId()).toString(),servers,deadServers,
3533       ProtobufUtil.toServerName(proto.getMaster()),backupMasters,rit,masterCoprocessors,
3534       proto.getBalancerOn());
3535   }
3536
3537   /**
3538    * Convert a ClusterStatus to a protobuf ClusterStatus
3539    *
3540    * @return the protobuf ClusterStatus
3541    */
3542   public static ClusterStatusProtos.ClusterStatus convert(ClusterStatus status) {
3543     ClusterStatusProtos.ClusterStatus.Builder builder =
3544         ClusterStatusProtos.ClusterStatus.newBuilder();
3545     builder
3546         .setHbaseVersion(HBaseVersionFileContent.newBuilder().setVersion(status.getHBaseVersion()));
3547
3548     if (status.getServers() != null) {
3549       for (ServerName serverName : status.getServers()) {
3550         LiveServerInfo.Builder lsi =
3551             LiveServerInfo.newBuilder().setServer(ProtobufUtil.toServerName(serverName));
3552         status.getLoad(serverName);
3553         lsi.setServerLoad(status.getLoad(serverName).obtainServerLoadPB());
3554         builder.addLiveServers(lsi.build());
3555       }
3556     }
3557
3558     if (status.getDeadServerNames() != null) {
3559       for (ServerName deadServer : status.getDeadServerNames()) {
3560         builder.addDeadServers(ProtobufUtil.toServerName(deadServer));
3561       }
3562     }
3563
3564     if (status.getRegionsInTransition() != null) {
3565       for (RegionState rit : status.getRegionsInTransition()) {
3566         ClusterStatusProtos.RegionState rs = rit.convert();
3567         RegionSpecifier.Builder spec =
3568             RegionSpecifier.newBuilder().setType(RegionSpecifierType.REGION_NAME);
3569         spec.setValue(ByteStringer.wrap(rit.getRegion().getRegionName()));
3570
3571         RegionInTransition pbRIT =
3572             RegionInTransition.newBuilder().setSpec(spec.build()).setRegionState(rs).build();
3573         builder.addRegionsInTransition(pbRIT);
3574       }
3575     }
3576
3577     if (status.getClusterId() != null) {
3578       builder.setClusterId(new ClusterId(status.getClusterId()).convert());
3579     }
3580
3581     if (status.getMasterCoprocessors() != null) {
3582       for (String coprocessor : status.getMasterCoprocessors()) {
3583         builder.addMasterCoprocessors(HBaseProtos.Coprocessor.newBuilder().setName(coprocessor));
3584       }
3585     }
3586
3587     if (status.getMaster() != null) {
3588       builder.setMaster(ProtobufUtil.toServerName(status.getMaster()));
3589     }
3590
3591     if (status.getBackupMasters() != null) {
3592       for (ServerName backup : status.getBackupMasters()) {
3593         builder.addBackupMasters(ProtobufUtil.toServerName(backup));
3594       }
3595     }
3596
3597     if (status.getBalancerOn() != null) {
3598       builder.setBalancerOn(status.getBalancerOn());
3599     }
3600
3601     return builder.build();
3602   }
3603
3604   public static RegionLoadStats createRegionLoadStats(ClientProtos.RegionLoadStats stats) {
3605     return new RegionLoadStats(stats.getMemstoreLoad(), stats.getHeapOccupancy(),
3606         stats.getCompactionPressure());
3607   }
3608 }