View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.protobuf;
19  
20  
21  import com.google.common.collect.ArrayListMultimap;
22  import com.google.common.collect.ListMultimap;
23  import com.google.common.collect.Lists;
24  import com.google.protobuf.ByteString;
25  import com.google.protobuf.InvalidProtocolBufferException;
26  import com.google.protobuf.Message;
27  import com.google.protobuf.Parser;
28  import com.google.protobuf.RpcChannel;
29  import com.google.protobuf.Service;
30  import com.google.protobuf.ServiceException;
31  import com.google.protobuf.TextFormat;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.Cell;
35  import org.apache.hadoop.hbase.CellScanner;
36  import org.apache.hadoop.hbase.CellUtil;
37  import org.apache.hadoop.hbase.DoNotRetryIOException;
38  import org.apache.hadoop.hbase.HBaseConfiguration;
39  import org.apache.hadoop.hbase.HConstants;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.HTableDescriptor;
42  import org.apache.hadoop.hbase.KeyValue;
43  import org.apache.hadoop.hbase.KeyValueUtil;
44  import org.apache.hadoop.hbase.NamespaceDescriptor;
45  import org.apache.hadoop.hbase.ServerName;
46  import org.apache.hadoop.hbase.TableName;
47  import org.apache.hadoop.hbase.Tag;
48  import org.apache.hadoop.hbase.client.Append;
49  import org.apache.hadoop.hbase.client.Consistency;
50  import org.apache.hadoop.hbase.client.Delete;
51  import org.apache.hadoop.hbase.client.Durability;
52  import org.apache.hadoop.hbase.client.Get;
53  import org.apache.hadoop.hbase.client.Increment;
54  import org.apache.hadoop.hbase.client.Mutation;
55  import org.apache.hadoop.hbase.client.Put;
56  import org.apache.hadoop.hbase.client.Result;
57  import org.apache.hadoop.hbase.client.Scan;
58  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
59  import org.apache.hadoop.hbase.exceptions.DeserializationException;
60  import org.apache.hadoop.hbase.filter.ByteArrayComparable;
61  import org.apache.hadoop.hbase.filter.Filter;
62  import org.apache.hadoop.hbase.io.TimeRange;
63  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
64  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService;
65  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
66  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
67  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
68  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
69  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
70  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
71  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
72  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
73  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
74  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
75  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
76  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
77  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
78  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
79  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
80  import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
81  import org.apache.hadoop.hbase.protobuf.generated.CellProtos;
82  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
83  import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
84  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
85  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
86  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
87  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Column;
88  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall;
89  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
90  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
91  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
92  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
93  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
94  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue;
95  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue.QualifierValue;
96  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.DeleteType;
97  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
98  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
99  import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
100 import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
101 import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
102 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
103 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
104 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo;
105 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
106 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
107 import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
108 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
109 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
110 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService;
111 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
112 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
113 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
114 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
115 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
116 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
117 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor.EventType;
118 import org.apache.hadoop.hbase.security.access.Permission;
119 import org.apache.hadoop.hbase.security.access.TablePermission;
120 import org.apache.hadoop.hbase.security.access.UserPermission;
121 import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier;
122 import org.apache.hadoop.hbase.security.visibility.Authorizations;
123 import org.apache.hadoop.hbase.security.visibility.CellVisibility;
124 import org.apache.hadoop.hbase.util.ByteStringer;
125 import org.apache.hadoop.hbase.util.Bytes;
126 import org.apache.hadoop.hbase.util.DynamicClassLoader;
127 import org.apache.hadoop.hbase.util.ExceptionUtil;
128 import org.apache.hadoop.hbase.util.Methods;
129 import org.apache.hadoop.hbase.util.Pair;
130 import org.apache.hadoop.io.Text;
131 import org.apache.hadoop.ipc.RemoteException;
132 import org.apache.hadoop.security.token.Token;
133 
134 import java.io.ByteArrayOutputStream;
135 import java.io.IOException;
136 import java.lang.reflect.Constructor;
137 import java.lang.reflect.InvocationTargetException;
138 import java.lang.reflect.Method;
139 import java.lang.reflect.ParameterizedType;
140 import java.lang.reflect.Type;
141 import java.nio.ByteBuffer;
142 import java.util.ArrayList;
143 import java.util.Collection;
144 import java.util.HashMap;
145 import java.util.List;
146 import java.util.Map;
147 import java.util.Map.Entry;
148 import java.util.NavigableSet;
149 
150 import static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME;
151 
152 /**
153  * Protobufs utility.
154  */
155 public final class ProtobufUtil {
156 
157   private ProtobufUtil() {
158   }
159 
160   /**
161    * Primitive type to class mapping.
162    */
163   private final static Map<String, Class<?>>
164     PRIMITIVES = new HashMap<String, Class<?>>();
165 
166 
167   /**
168    * Many results are simple: no cell, exists true or false. To save on object creations,
169    *  we reuse them across calls.
170    */
171   private final static Cell[] EMPTY_CELL_ARRAY = new Cell[]{};
172   private final static Result EMPTY_RESULT = Result.create(EMPTY_CELL_ARRAY);
173   private final static Result EMPTY_RESULT_EXISTS_TRUE = Result.create(null, true);
174   private final static Result EMPTY_RESULT_EXISTS_FALSE = Result.create(null, false);
175   private final static Result EMPTY_RESULT_STALE = Result.create(EMPTY_CELL_ARRAY, null, true);
176   private final static Result EMPTY_RESULT_EXISTS_TRUE_STALE
177     = Result.create((Cell[])null, true, true);
178   private final static Result EMPTY_RESULT_EXISTS_FALSE_STALE
179     = Result.create((Cell[])null, false, true);
180 
181   private final static ClientProtos.Result EMPTY_RESULT_PB;
182   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_TRUE;
183   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_FALSE;
184   private final static ClientProtos.Result EMPTY_RESULT_PB_STALE;
185   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_TRUE_STALE;
186   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_FALSE_STALE;
187 
188 
189   static {
190     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
191 
192     builder.setExists(true);
193     builder.setAssociatedCellCount(0);
194     EMPTY_RESULT_PB_EXISTS_TRUE =  builder.build();
195 
196     builder.setStale(true);
197     EMPTY_RESULT_PB_EXISTS_TRUE_STALE = builder.build();
198     builder.clear();
199 
200     builder.setExists(false);
201     builder.setAssociatedCellCount(0);
202     EMPTY_RESULT_PB_EXISTS_FALSE =  builder.build();
203     builder.setStale(true);
204     EMPTY_RESULT_PB_EXISTS_FALSE_STALE = builder.build();
205 
206     builder.clear();
207     builder.setAssociatedCellCount(0);
208     EMPTY_RESULT_PB =  builder.build();
209     builder.setStale(true);
210     EMPTY_RESULT_PB_STALE = builder.build();
211   }
212 
213   /**
214    * Dynamic class loader to load filter/comparators
215    */
216   private final static ClassLoader CLASS_LOADER;
217 
218   static {
219     ClassLoader parent = ProtobufUtil.class.getClassLoader();
220     Configuration conf = HBaseConfiguration.create();
221     CLASS_LOADER = new DynamicClassLoader(conf, parent);
222 
223     PRIMITIVES.put(Boolean.TYPE.getName(), Boolean.TYPE);
224     PRIMITIVES.put(Byte.TYPE.getName(), Byte.TYPE);
225     PRIMITIVES.put(Character.TYPE.getName(), Character.TYPE);
226     PRIMITIVES.put(Short.TYPE.getName(), Short.TYPE);
227     PRIMITIVES.put(Integer.TYPE.getName(), Integer.TYPE);
228     PRIMITIVES.put(Long.TYPE.getName(), Long.TYPE);
229     PRIMITIVES.put(Float.TYPE.getName(), Float.TYPE);
230     PRIMITIVES.put(Double.TYPE.getName(), Double.TYPE);
231     PRIMITIVES.put(Void.TYPE.getName(), Void.TYPE);
232   }
233 
234   /**
235    * Magic we put ahead of a serialized protobuf message.
236    * For example, all znode content is protobuf messages with the below magic
237    * for preamble.
238    */
239   public static final byte [] PB_MAGIC = new byte [] {'P', 'B', 'U', 'F'};
240   private static final String PB_MAGIC_STR = Bytes.toString(PB_MAGIC);
241 
242   /**
243    * Prepend the passed bytes with four bytes of magic, {@link #PB_MAGIC}, to flag what
244    * follows as a protobuf in hbase.  Prepend these bytes to all content written to znodes, etc.
245    * @param bytes Bytes to decorate
246    * @return The passed <code>bytes</codes> with magic prepended (Creates a new
247    * byte array that is <code>bytes.length</code> plus {@link #PB_MAGIC}.length.
248    */
249   public static byte [] prependPBMagic(final byte [] bytes) {
250     return Bytes.add(PB_MAGIC, bytes);
251   }
252 
253   /**
254    * @param bytes Bytes to check.
255    * @return True if passed <code>bytes</code> has {@link #PB_MAGIC} for a prefix.
256    */
257   public static boolean isPBMagicPrefix(final byte [] bytes) {
258     return isPBMagicPrefix(bytes, 0, bytes.length);
259   }
260 
261   /**
262    * @param bytes Bytes to check.
263    * @return True if passed <code>bytes</code> has {@link #PB_MAGIC} for a prefix.
264    */
265   public static boolean isPBMagicPrefix(final byte [] bytes, int offset, int len) {
266     if (bytes == null || len < PB_MAGIC.length) return false;
267     return Bytes.compareTo(PB_MAGIC, 0, PB_MAGIC.length, bytes, offset, PB_MAGIC.length) == 0;
268   }
269 
270   /**
271    * @param bytes
272    * @throws DeserializationException if we are missing the pb magic prefix
273    */
274   public static void expectPBMagicPrefix(final byte [] bytes) throws DeserializationException {
275     if (!isPBMagicPrefix(bytes)) {
276       throw new DeserializationException("Missing pb magic " + PB_MAGIC_STR + " prefix");
277     }
278   }
279 
280   /**
281    * @return Length of {@link #PB_MAGIC}
282    */
283   public static int lengthOfPBMagic() {
284     return PB_MAGIC.length;
285   }
286 
287   /**
288    * Return the IOException thrown by the remote server wrapped in
289    * ServiceException as cause.
290    *
291    * @param se ServiceException that wraps IO exception thrown by the server
292    * @return Exception wrapped in ServiceException or
293    *   a new IOException that wraps the unexpected ServiceException.
294    */
295   public static IOException getRemoteException(ServiceException se) {
296     Throwable e = se.getCause();
297     if (e == null) {
298       return new IOException(se);
299     }
300     if (ExceptionUtil.isInterrupt(e)) {
301       return ExceptionUtil.asInterrupt(e);
302     }
303     if (e instanceof RemoteException) {
304       e = ((RemoteException) e).unwrapRemoteException();
305     }
306     return e instanceof IOException ? (IOException) e : new IOException(se);
307   }
308 
309   /**
310    * Convert a ServerName to a protocol buffer ServerName
311    *
312    * @param serverName the ServerName to convert
313    * @return the converted protocol buffer ServerName
314    * @see #toServerName(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName)
315    */
316   public static HBaseProtos.ServerName
317       toServerName(final ServerName serverName) {
318     if (serverName == null) return null;
319     HBaseProtos.ServerName.Builder builder =
320       HBaseProtos.ServerName.newBuilder();
321     builder.setHostName(serverName.getHostname());
322     if (serverName.getPort() >= 0) {
323       builder.setPort(serverName.getPort());
324     }
325     if (serverName.getStartcode() >= 0) {
326       builder.setStartCode(serverName.getStartcode());
327     }
328     return builder.build();
329   }
330 
331   /**
332    * Convert a protocol buffer ServerName to a ServerName
333    *
334    * @param proto the protocol buffer ServerName to convert
335    * @return the converted ServerName
336    */
337   public static ServerName toServerName(final HBaseProtos.ServerName proto) {
338     if (proto == null) return null;
339     String hostName = proto.getHostName();
340     long startCode = -1;
341     int port = -1;
342     if (proto.hasPort()) {
343       port = proto.getPort();
344     }
345     if (proto.hasStartCode()) {
346       startCode = proto.getStartCode();
347     }
348     return ServerName.valueOf(hostName, port, startCode);
349   }
350 
351   /**
352    * Get HTableDescriptor[] from GetTableDescriptorsResponse protobuf
353    *
354    * @param proto the GetTableDescriptorsResponse
355    * @return HTableDescriptor[]
356    */
357   public static HTableDescriptor[] getHTableDescriptorArray(GetTableDescriptorsResponse proto) {
358     if (proto == null) return null;
359 
360     HTableDescriptor[] ret = new HTableDescriptor[proto.getTableSchemaCount()];
361     for (int i = 0; i < proto.getTableSchemaCount(); ++i) {
362       ret[i] = HTableDescriptor.convert(proto.getTableSchema(i));
363     }
364     return ret;
365   }
366 
367   /**
368    * get the split keys in form "byte [][]" from a CreateTableRequest proto
369    *
370    * @param proto the CreateTableRequest
371    * @return the split keys
372    */
373   public static byte [][] getSplitKeysArray(final CreateTableRequest proto) {
374     byte [][] splitKeys = new byte[proto.getSplitKeysCount()][];
375     for (int i = 0; i < proto.getSplitKeysCount(); ++i) {
376       splitKeys[i] = proto.getSplitKeys(i).toByteArray();
377     }
378     return splitKeys;
379   }
380 
381   /**
382    * Convert a protobuf Durability into a client Durability
383    */
384   public static Durability toDurability(
385       final ClientProtos.MutationProto.Durability proto) {
386     switch(proto) {
387     case USE_DEFAULT:
388       return Durability.USE_DEFAULT;
389     case SKIP_WAL:
390       return Durability.SKIP_WAL;
391     case ASYNC_WAL:
392       return Durability.ASYNC_WAL;
393     case SYNC_WAL:
394       return Durability.SYNC_WAL;
395     case FSYNC_WAL:
396       return Durability.FSYNC_WAL;
397     default:
398       return Durability.USE_DEFAULT;
399     }
400   }
401 
402   /**
403    * Convert a client Durability into a protbuf Durability
404    */
405   public static ClientProtos.MutationProto.Durability toDurability(
406       final Durability d) {
407     switch(d) {
408     case USE_DEFAULT:
409       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
410     case SKIP_WAL:
411       return ClientProtos.MutationProto.Durability.SKIP_WAL;
412     case ASYNC_WAL:
413       return ClientProtos.MutationProto.Durability.ASYNC_WAL;
414     case SYNC_WAL:
415       return ClientProtos.MutationProto.Durability.SYNC_WAL;
416     case FSYNC_WAL:
417       return ClientProtos.MutationProto.Durability.FSYNC_WAL;
418     default:
419       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
420     }
421   }
422 
423   /**
424    * Convert a protocol buffer Get to a client Get
425    *
426    * @param proto the protocol buffer Get to convert
427    * @return the converted client Get
428    * @throws IOException
429    */
430   public static Get toGet(
431       final ClientProtos.Get proto) throws IOException {
432     if (proto == null) return null;
433     byte[] row = proto.getRow().toByteArray();
434     Get get = new Get(row);
435     if (proto.hasCacheBlocks()) {
436       get.setCacheBlocks(proto.getCacheBlocks());
437     }
438     if (proto.hasMaxVersions()) {
439       get.setMaxVersions(proto.getMaxVersions());
440     }
441     if (proto.hasStoreLimit()) {
442       get.setMaxResultsPerColumnFamily(proto.getStoreLimit());
443     }
444     if (proto.hasStoreOffset()) {
445       get.setRowOffsetPerColumnFamily(proto.getStoreOffset());
446     }
447     if (proto.hasTimeRange()) {
448       HBaseProtos.TimeRange timeRange = proto.getTimeRange();
449       long minStamp = 0;
450       long maxStamp = Long.MAX_VALUE;
451       if (timeRange.hasFrom()) {
452         minStamp = timeRange.getFrom();
453       }
454       if (timeRange.hasTo()) {
455         maxStamp = timeRange.getTo();
456       }
457       get.setTimeRange(minStamp, maxStamp);
458     }
459     if (proto.hasFilter()) {
460       FilterProtos.Filter filter = proto.getFilter();
461       get.setFilter(ProtobufUtil.toFilter(filter));
462     }
463     for (NameBytesPair attribute: proto.getAttributeList()) {
464       get.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
465     }
466     if (proto.getColumnCount() > 0) {
467       for (Column column: proto.getColumnList()) {
468         byte[] family = column.getFamily().toByteArray();
469         if (column.getQualifierCount() > 0) {
470           for (ByteString qualifier: column.getQualifierList()) {
471             get.addColumn(family, qualifier.toByteArray());
472           }
473         } else {
474           get.addFamily(family);
475         }
476       }
477     }
478     if (proto.hasExistenceOnly() && proto.getExistenceOnly()){
479       get.setCheckExistenceOnly(true);
480     }
481     if (proto.hasClosestRowBefore() && proto.getClosestRowBefore()){
482       get.setClosestRowBefore(true);
483     }
484     if (proto.hasConsistency()) {
485       get.setConsistency(toConsistency(proto.getConsistency()));
486     }
487     return get;
488   }
489 
490   public static Consistency toConsistency(ClientProtos.Consistency consistency) {
491     switch (consistency) {
492       case STRONG : return Consistency.STRONG;
493       case TIMELINE : return Consistency.TIMELINE;
494       default : return Consistency.STRONG;
495     }
496   }
497 
498   public static ClientProtos.Consistency toConsistency(Consistency consistency) {
499     switch (consistency) {
500       case STRONG : return ClientProtos.Consistency.STRONG;
501       case TIMELINE : return ClientProtos.Consistency.TIMELINE;
502       default : return ClientProtos.Consistency.STRONG;
503     }
504   }
505 
506   /**
507    * Convert a protocol buffer Mutate to a Put.
508    *
509    * @param proto The protocol buffer MutationProto to convert
510    * @return A client Put.
511    * @throws IOException
512    */
513   public static Put toPut(final MutationProto proto)
514   throws IOException {
515     return toPut(proto, null);
516   }
517 
518   /**
519    * Convert a protocol buffer Mutate to a Put.
520    *
521    * @param proto The protocol buffer MutationProto to convert
522    * @param cellScanner If non-null, the Cell data that goes with this proto.
523    * @return A client Put.
524    * @throws IOException
525    */
526   public static Put toPut(final MutationProto proto, final CellScanner cellScanner)
527   throws IOException {
528     // TODO: Server-side at least why do we convert back to the Client types?  Why not just pb it?
529     MutationType type = proto.getMutateType();
530     assert type == MutationType.PUT: type.name();
531     long timestamp = proto.hasTimestamp()? proto.getTimestamp(): HConstants.LATEST_TIMESTAMP;
532     Put put = null;
533     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
534     if (cellCount > 0) {
535       // The proto has metadata only and the data is separate to be found in the cellScanner.
536       if (cellScanner == null) {
537         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
538             toShortString(proto));
539       }
540       for (int i = 0; i < cellCount; i++) {
541         if (!cellScanner.advance()) {
542           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
543             " no cell returned: " + toShortString(proto));
544         }
545         Cell cell = cellScanner.current();
546         if (put == null) {
547           put = new Put(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
548         }
549         put.add(cell);
550       }
551     } else {
552       if (proto.hasRow()) {
553         put = new Put(proto.getRow().asReadOnlyByteBuffer(), timestamp);
554       } else {
555         throw new IllegalArgumentException("row cannot be null");
556       }
557       // The proto has the metadata and the data itself
558       for (ColumnValue column: proto.getColumnValueList()) {
559         byte[] family = column.getFamily().toByteArray();
560         for (QualifierValue qv: column.getQualifierValueList()) {
561           if (!qv.hasValue()) {
562             throw new DoNotRetryIOException(
563                 "Missing required field: qualifier value");
564           }
565           ByteBuffer qualifier =
566               qv.hasQualifier() ? qv.getQualifier().asReadOnlyByteBuffer() : null;
567           ByteBuffer value =
568               qv.hasValue() ? qv.getValue().asReadOnlyByteBuffer() : null;
569           long ts = timestamp;
570           if (qv.hasTimestamp()) {
571             ts = qv.getTimestamp();
572           }
573           byte[] tags;
574           if (qv.hasTags()) {
575             tags = qv.getTags().toByteArray();
576             Object[] array = Tag.asList(tags, 0, (short)tags.length).toArray();
577             Tag[] tagArray = new Tag[array.length];
578             for(int i = 0; i< array.length; i++) {
579               tagArray[i] = (Tag)array[i];
580             }
581             if(qv.hasDeleteType()) {
582               byte[] qual = qv.hasQualifier() ? qv.getQualifier().toByteArray() : null;
583               put.add(new KeyValue(proto.getRow().toByteArray(), family, qual, ts,
584                   fromDeleteType(qv.getDeleteType()), null, tags));
585             } else {
586               put.addImmutable(family, qualifier, ts, value, tagArray);
587             }
588           } else {
589             if(qv.hasDeleteType()) {
590               byte[] qual = qv.hasQualifier() ? qv.getQualifier().toByteArray() : null;
591               put.add(new KeyValue(proto.getRow().toByteArray(), family, qual, ts,
592                   fromDeleteType(qv.getDeleteType())));
593             } else{
594               put.addImmutable(family, qualifier, ts, value);
595             }
596           }
597         }
598       }
599     }
600     put.setDurability(toDurability(proto.getDurability()));
601     for (NameBytesPair attribute: proto.getAttributeList()) {
602       put.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
603     }
604     return put;
605   }
606 
607   /**
608    * Convert a protocol buffer Mutate to a Delete
609    *
610    * @param proto the protocol buffer Mutate to convert
611    * @return the converted client Delete
612    * @throws IOException
613    */
614   public static Delete toDelete(final MutationProto proto)
615   throws IOException {
616     return toDelete(proto, null);
617   }
618 
619   /**
620    * Convert a protocol buffer Mutate to a Delete
621    *
622    * @param proto the protocol buffer Mutate to convert
623    * @param cellScanner if non-null, the data that goes with this delete.
624    * @return the converted client Delete
625    * @throws IOException
626    */
627   public static Delete toDelete(final MutationProto proto, final CellScanner cellScanner)
628   throws IOException {
629     MutationType type = proto.getMutateType();
630     assert type == MutationType.DELETE : type.name();
631     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
632     long timestamp = HConstants.LATEST_TIMESTAMP;
633     if (proto.hasTimestamp()) {
634       timestamp = proto.getTimestamp();
635     }
636     Delete delete = null;
637     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
638     if (cellCount > 0) {
639       // The proto has metadata only and the data is separate to be found in the cellScanner.
640       if (cellScanner == null) {
641         // TextFormat should be fine for a Delete since it carries no data, just coordinates.
642         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
643           TextFormat.shortDebugString(proto));
644       }
645       for (int i = 0; i < cellCount; i++) {
646         if (!cellScanner.advance()) {
647           // TextFormat should be fine for a Delete since it carries no data, just coordinates.
648           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
649             " no cell returned: " + TextFormat.shortDebugString(proto));
650         }
651         Cell cell = cellScanner.current();
652         if (delete == null) {
653           delete =
654             new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
655         }
656         delete.addDeleteMarker(KeyValueUtil.ensureKeyValue(cell));
657       }
658     } else {
659       delete = new Delete(row, timestamp);
660       for (ColumnValue column: proto.getColumnValueList()) {
661         byte[] family = column.getFamily().toByteArray();
662         for (QualifierValue qv: column.getQualifierValueList()) {
663           DeleteType deleteType = qv.getDeleteType();
664           byte[] qualifier = null;
665           if (qv.hasQualifier()) {
666             qualifier = qv.getQualifier().toByteArray();
667           }
668           long ts = HConstants.LATEST_TIMESTAMP;
669           if (qv.hasTimestamp()) {
670             ts = qv.getTimestamp();
671           }
672           if (deleteType == DeleteType.DELETE_ONE_VERSION) {
673             delete.deleteColumn(family, qualifier, ts);
674           } else if (deleteType == DeleteType.DELETE_MULTIPLE_VERSIONS) {
675             delete.deleteColumns(family, qualifier, ts);
676           } else if (deleteType == DeleteType.DELETE_FAMILY_VERSION) {
677             delete.deleteFamilyVersion(family, ts);
678           } else {
679             delete.deleteFamily(family, ts);
680           }
681         }
682       }
683     }
684     delete.setDurability(toDurability(proto.getDurability()));
685     for (NameBytesPair attribute: proto.getAttributeList()) {
686       delete.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
687     }
688     return delete;
689   }
690 
691   /**
692    * Convert a protocol buffer Mutate to an Append
693    * @param cellScanner
694    * @param proto the protocol buffer Mutate to convert
695    * @return the converted client Append
696    * @throws IOException
697    */
698   public static Append toAppend(final MutationProto proto, final CellScanner cellScanner)
699   throws IOException {
700     MutationType type = proto.getMutateType();
701     assert type == MutationType.APPEND : type.name();
702     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
703     Append append = null;
704     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
705     if (cellCount > 0) {
706       // The proto has metadata only and the data is separate to be found in the cellScanner.
707       if (cellScanner == null) {
708         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
709           toShortString(proto));
710       }
711       for (int i = 0; i < cellCount; i++) {
712         if (!cellScanner.advance()) {
713           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
714             " no cell returned: " + toShortString(proto));
715         }
716         Cell cell = cellScanner.current();
717         if (append == null) {
718           append = new Append(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
719         }
720         append.add(KeyValueUtil.ensureKeyValue(cell));
721       }
722     } else {
723       append = new Append(row);
724       for (ColumnValue column: proto.getColumnValueList()) {
725         byte[] family = column.getFamily().toByteArray();
726         for (QualifierValue qv: column.getQualifierValueList()) {
727           byte[] qualifier = qv.getQualifier().toByteArray();
728           if (!qv.hasValue()) {
729             throw new DoNotRetryIOException(
730               "Missing required field: qualifier value");
731           }
732           byte[] value = qv.getValue().toByteArray();
733           byte[] tags = null;
734           if (qv.hasTags()) {
735             tags = qv.getTags().toByteArray();
736           }
737           append.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
738               KeyValue.Type.Put, value, tags));
739         }
740       }
741     }
742     append.setDurability(toDurability(proto.getDurability()));
743     for (NameBytesPair attribute: proto.getAttributeList()) {
744       append.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
745     }
746     return append;
747   }
748 
749   /**
750    * Convert a MutateRequest to Mutation
751    *
752    * @param proto the protocol buffer Mutate to convert
753    * @return the converted Mutation
754    * @throws IOException
755    */
756   public static Mutation toMutation(final MutationProto proto) throws IOException {
757     MutationType type = proto.getMutateType();
758     if (type == MutationType.APPEND) {
759       return toAppend(proto, null);
760     }
761     if (type == MutationType.DELETE) {
762       return toDelete(proto, null);
763     }
764     if (type == MutationType.PUT) {
765       return toPut(proto, null);
766     }
767     throw new IOException("Unknown mutation type " + type);
768   }
769 
770   /**
771    * Convert a protocol buffer Mutate to an Increment
772    *
773    * @param proto the protocol buffer Mutate to convert
774    * @return the converted client Increment
775    * @throws IOException
776    */
777   public static Increment toIncrement(final MutationProto proto, final CellScanner cellScanner)
778   throws IOException {
779     MutationType type = proto.getMutateType();
780     assert type == MutationType.INCREMENT : type.name();
781     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
782     Increment increment = null;
783     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
784     if (cellCount > 0) {
785       // The proto has metadata only and the data is separate to be found in the cellScanner.
786       if (cellScanner == null) {
787         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
788           TextFormat.shortDebugString(proto));
789       }
790       for (int i = 0; i < cellCount; i++) {
791         if (!cellScanner.advance()) {
792           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
793             " no cell returned: " + TextFormat.shortDebugString(proto));
794         }
795         Cell cell = cellScanner.current();
796         if (increment == null) {
797           increment = new Increment(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
798         }
799         increment.add(KeyValueUtil.ensureKeyValue(cell));
800       }
801     } else {
802       increment = new Increment(row);
803       for (ColumnValue column: proto.getColumnValueList()) {
804         byte[] family = column.getFamily().toByteArray();
805         for (QualifierValue qv: column.getQualifierValueList()) {
806           byte[] qualifier = qv.getQualifier().toByteArray();
807           if (!qv.hasValue()) {
808             throw new DoNotRetryIOException("Missing required field: qualifier value");
809           }
810           byte[] value = qv.getValue().toByteArray();
811           byte[] tags = null;
812           if (qv.hasTags()) {
813             tags = qv.getTags().toByteArray();
814           }
815           increment.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
816               KeyValue.Type.Put, value, tags));
817         }
818       }
819     }
820     if (proto.hasTimeRange()) {
821       HBaseProtos.TimeRange timeRange = proto.getTimeRange();
822       long minStamp = 0;
823       long maxStamp = Long.MAX_VALUE;
824       if (timeRange.hasFrom()) {
825         minStamp = timeRange.getFrom();
826       }
827       if (timeRange.hasTo()) {
828         maxStamp = timeRange.getTo();
829       }
830       increment.setTimeRange(minStamp, maxStamp);
831     }
832     increment.setDurability(toDurability(proto.getDurability()));
833     for (NameBytesPair attribute : proto.getAttributeList()) {
834       increment.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
835     }
836     return increment;
837   }
838 
839   /**
840    * Convert a client Scan to a protocol buffer Scan
841    *
842    * @param scan the client Scan to convert
843    * @return the converted protocol buffer Scan
844    * @throws IOException
845    */
846   public static ClientProtos.Scan toScan(
847       final Scan scan) throws IOException {
848     ClientProtos.Scan.Builder scanBuilder =
849       ClientProtos.Scan.newBuilder();
850     scanBuilder.setCacheBlocks(scan.getCacheBlocks());
851     if (scan.getBatch() > 0) {
852       scanBuilder.setBatchSize(scan.getBatch());
853     }
854     if (scan.getMaxResultSize() > 0) {
855       scanBuilder.setMaxResultSize(scan.getMaxResultSize());
856     }
857     if (scan.isSmall()) {
858       scanBuilder.setSmall(scan.isSmall());
859     }
860     Boolean loadColumnFamiliesOnDemand = scan.getLoadColumnFamiliesOnDemandValue();
861     if (loadColumnFamiliesOnDemand != null) {
862       scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand.booleanValue());
863     }
864     scanBuilder.setMaxVersions(scan.getMaxVersions());
865     TimeRange timeRange = scan.getTimeRange();
866     if (!timeRange.isAllTime()) {
867       HBaseProtos.TimeRange.Builder timeRangeBuilder =
868         HBaseProtos.TimeRange.newBuilder();
869       timeRangeBuilder.setFrom(timeRange.getMin());
870       timeRangeBuilder.setTo(timeRange.getMax());
871       scanBuilder.setTimeRange(timeRangeBuilder.build());
872     }
873     Map<String, byte[]> attributes = scan.getAttributesMap();
874     if (!attributes.isEmpty()) {
875       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
876       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
877         attributeBuilder.setName(attribute.getKey());
878         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
879         scanBuilder.addAttribute(attributeBuilder.build());
880       }
881     }
882     byte[] startRow = scan.getStartRow();
883     if (startRow != null && startRow.length > 0) {
884       scanBuilder.setStartRow(ByteStringer.wrap(startRow));
885     }
886     byte[] stopRow = scan.getStopRow();
887     if (stopRow != null && stopRow.length > 0) {
888       scanBuilder.setStopRow(ByteStringer.wrap(stopRow));
889     }
890     if (scan.hasFilter()) {
891       scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
892     }
893     if (scan.hasFamilies()) {
894       Column.Builder columnBuilder = Column.newBuilder();
895       for (Map.Entry<byte[],NavigableSet<byte []>>
896           family: scan.getFamilyMap().entrySet()) {
897         columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
898         NavigableSet<byte []> qualifiers = family.getValue();
899         columnBuilder.clearQualifier();
900         if (qualifiers != null && qualifiers.size() > 0) {
901           for (byte [] qualifier: qualifiers) {
902             columnBuilder.addQualifier(ByteStringer.wrap(qualifier));
903           }
904         }
905         scanBuilder.addColumn(columnBuilder.build());
906       }
907     }
908     if (scan.getMaxResultsPerColumnFamily() >= 0) {
909       scanBuilder.setStoreLimit(scan.getMaxResultsPerColumnFamily());
910     }
911     if (scan.getRowOffsetPerColumnFamily() > 0) {
912       scanBuilder.setStoreOffset(scan.getRowOffsetPerColumnFamily());
913     }
914     if (scan.isReversed()) {
915       scanBuilder.setReversed(scan.isReversed());
916     }
917     if (scan.getConsistency() == Consistency.TIMELINE) {
918       scanBuilder.setConsistency(toConsistency(scan.getConsistency()));
919     }
920     if (scan.getCaching() > 0) {
921       scanBuilder.setCaching(scan.getCaching());
922     }
923     return scanBuilder.build();
924   }
925 
926   /**
927    * Convert a protocol buffer Scan to a client Scan
928    *
929    * @param proto the protocol buffer Scan to convert
930    * @return the converted client Scan
931    * @throws IOException
932    */
933   public static Scan toScan(
934       final ClientProtos.Scan proto) throws IOException {
935     byte [] startRow = HConstants.EMPTY_START_ROW;
936     byte [] stopRow  = HConstants.EMPTY_END_ROW;
937     if (proto.hasStartRow()) {
938       startRow = proto.getStartRow().toByteArray();
939     }
940     if (proto.hasStopRow()) {
941       stopRow = proto.getStopRow().toByteArray();
942     }
943     Scan scan = new Scan(startRow, stopRow);
944     if (proto.hasCacheBlocks()) {
945       scan.setCacheBlocks(proto.getCacheBlocks());
946     }
947     if (proto.hasMaxVersions()) {
948       scan.setMaxVersions(proto.getMaxVersions());
949     }
950     if (proto.hasStoreLimit()) {
951       scan.setMaxResultsPerColumnFamily(proto.getStoreLimit());
952     }
953     if (proto.hasStoreOffset()) {
954       scan.setRowOffsetPerColumnFamily(proto.getStoreOffset());
955     }
956     if (proto.hasLoadColumnFamiliesOnDemand()) {
957       scan.setLoadColumnFamiliesOnDemand(proto.getLoadColumnFamiliesOnDemand());
958     }
959     if (proto.hasTimeRange()) {
960       HBaseProtos.TimeRange timeRange = proto.getTimeRange();
961       long minStamp = 0;
962       long maxStamp = Long.MAX_VALUE;
963       if (timeRange.hasFrom()) {
964         minStamp = timeRange.getFrom();
965       }
966       if (timeRange.hasTo()) {
967         maxStamp = timeRange.getTo();
968       }
969       scan.setTimeRange(minStamp, maxStamp);
970     }
971     if (proto.hasFilter()) {
972       FilterProtos.Filter filter = proto.getFilter();
973       scan.setFilter(ProtobufUtil.toFilter(filter));
974     }
975     if (proto.hasBatchSize()) {
976       scan.setBatch(proto.getBatchSize());
977     }
978     if (proto.hasMaxResultSize()) {
979       scan.setMaxResultSize(proto.getMaxResultSize());
980     }
981     if (proto.hasSmall()) {
982       scan.setSmall(proto.getSmall());
983     }
984     for (NameBytesPair attribute: proto.getAttributeList()) {
985       scan.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
986     }
987     if (proto.getColumnCount() > 0) {
988       for (Column column: proto.getColumnList()) {
989         byte[] family = column.getFamily().toByteArray();
990         if (column.getQualifierCount() > 0) {
991           for (ByteString qualifier: column.getQualifierList()) {
992             scan.addColumn(family, qualifier.toByteArray());
993           }
994         } else {
995           scan.addFamily(family);
996         }
997       }
998     }
999     if (proto.hasReversed()) {
1000       scan.setReversed(proto.getReversed());
1001     }
1002     if (proto.hasConsistency()) {
1003       scan.setConsistency(toConsistency(proto.getConsistency()));
1004     }
1005     if (proto.hasCaching()) {
1006       scan.setCaching(proto.getCaching());
1007     }
1008     return scan;
1009   }
1010 
1011   /**
1012    * Create a protocol buffer Get based on a client Get.
1013    *
1014    * @param get the client Get
1015    * @return a protocol buffer Get
1016    * @throws IOException
1017    */
1018   public static ClientProtos.Get toGet(
1019       final Get get) throws IOException {
1020     ClientProtos.Get.Builder builder =
1021       ClientProtos.Get.newBuilder();
1022     builder.setRow(ByteStringer.wrap(get.getRow()));
1023     builder.setCacheBlocks(get.getCacheBlocks());
1024     builder.setMaxVersions(get.getMaxVersions());
1025     if (get.getFilter() != null) {
1026       builder.setFilter(ProtobufUtil.toFilter(get.getFilter()));
1027     }
1028     TimeRange timeRange = get.getTimeRange();
1029     if (!timeRange.isAllTime()) {
1030       HBaseProtos.TimeRange.Builder timeRangeBuilder =
1031         HBaseProtos.TimeRange.newBuilder();
1032       timeRangeBuilder.setFrom(timeRange.getMin());
1033       timeRangeBuilder.setTo(timeRange.getMax());
1034       builder.setTimeRange(timeRangeBuilder.build());
1035     }
1036     Map<String, byte[]> attributes = get.getAttributesMap();
1037     if (!attributes.isEmpty()) {
1038       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1039       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
1040         attributeBuilder.setName(attribute.getKey());
1041         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1042         builder.addAttribute(attributeBuilder.build());
1043       }
1044     }
1045     if (get.hasFamilies()) {
1046       Column.Builder columnBuilder = Column.newBuilder();
1047       Map<byte[], NavigableSet<byte[]>> families = get.getFamilyMap();
1048       for (Map.Entry<byte[], NavigableSet<byte[]>> family: families.entrySet()) {
1049         NavigableSet<byte[]> qualifiers = family.getValue();
1050         columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1051         columnBuilder.clearQualifier();
1052         if (qualifiers != null && qualifiers.size() > 0) {
1053           for (byte[] qualifier: qualifiers) {
1054             columnBuilder.addQualifier(ByteStringer.wrap(qualifier));
1055           }
1056         }
1057         builder.addColumn(columnBuilder.build());
1058       }
1059     }
1060     if (get.getMaxResultsPerColumnFamily() >= 0) {
1061       builder.setStoreLimit(get.getMaxResultsPerColumnFamily());
1062     }
1063     if (get.getRowOffsetPerColumnFamily() > 0) {
1064       builder.setStoreOffset(get.getRowOffsetPerColumnFamily());
1065     }
1066     if (get.isCheckExistenceOnly()){
1067       builder.setExistenceOnly(true);
1068     }
1069     if (get.isClosestRowBefore()){
1070       builder.setClosestRowBefore(true);
1071     }
1072     if (get.getConsistency() != null && get.getConsistency() != Consistency.STRONG) {
1073       builder.setConsistency(toConsistency(get.getConsistency()));
1074     }
1075 
1076     return builder.build();
1077   }
1078 
1079   /**
1080    * Convert a client Increment to a protobuf Mutate.
1081    *
1082    * @param increment
1083    * @return the converted mutate
1084    */
1085   public static MutationProto toMutation(
1086     final Increment increment, final MutationProto.Builder builder, long nonce) {
1087     builder.setRow(ByteStringer.wrap(increment.getRow()));
1088     builder.setMutateType(MutationType.INCREMENT);
1089     builder.setDurability(toDurability(increment.getDurability()));
1090     if (nonce != HConstants.NO_NONCE) {
1091       builder.setNonce(nonce);
1092     }
1093     TimeRange timeRange = increment.getTimeRange();
1094     if (!timeRange.isAllTime()) {
1095       HBaseProtos.TimeRange.Builder timeRangeBuilder =
1096         HBaseProtos.TimeRange.newBuilder();
1097       timeRangeBuilder.setFrom(timeRange.getMin());
1098       timeRangeBuilder.setTo(timeRange.getMax());
1099       builder.setTimeRange(timeRangeBuilder.build());
1100     }
1101     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1102     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1103     for (Map.Entry<byte[], List<Cell>> family: increment.getFamilyCellMap().entrySet()) {
1104       columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1105       columnBuilder.clearQualifierValue();
1106       List<Cell> values = family.getValue();
1107       if (values != null && values.size() > 0) {
1108         for (Cell cell: values) {
1109           KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
1110           valueBuilder.setQualifier(ByteStringer.wrap(
1111               kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()));
1112           valueBuilder.setValue(ByteStringer.wrap(
1113               kv.getValueArray(), kv.getValueOffset(), kv.getValueLength()));
1114           if (kv.getTagsLength() > 0) {
1115             valueBuilder.setTags(ByteStringer.wrap(kv.getTagsArray(),
1116                 kv.getTagsOffset(), kv.getTagsLength()));
1117           }
1118           columnBuilder.addQualifierValue(valueBuilder.build());
1119         }
1120       }
1121       builder.addColumnValue(columnBuilder.build());
1122     }
1123     Map<String, byte[]> attributes = increment.getAttributesMap();
1124     if (!attributes.isEmpty()) {
1125       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1126       for (Map.Entry<String, byte[]> attribute : attributes.entrySet()) {
1127         attributeBuilder.setName(attribute.getKey());
1128         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1129         builder.addAttribute(attributeBuilder.build());
1130       }
1131     }
1132     return builder.build();
1133   }
1134 
1135   public static MutationProto toMutation(final MutationType type, final Mutation mutation)
1136     throws IOException {
1137     return toMutation(type, mutation, HConstants.NO_NONCE);
1138   }
1139 
1140   /**
1141    * Create a protocol buffer Mutate based on a client Mutation
1142    *
1143    * @param type
1144    * @param mutation
1145    * @return a protobuf'd Mutation
1146    * @throws IOException
1147    */
1148   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1149     final long nonce) throws IOException {
1150     return toMutation(type, mutation, MutationProto.newBuilder(), nonce);
1151   }
1152 
1153   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1154       MutationProto.Builder builder) throws IOException {
1155     return toMutation(type, mutation, builder, HConstants.NO_NONCE);
1156   }
1157 
1158   @SuppressWarnings("deprecation")
1159   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1160       MutationProto.Builder builder, long nonce)
1161   throws IOException {
1162     builder = getMutationBuilderAndSetCommonFields(type, mutation, builder);
1163     if (nonce != HConstants.NO_NONCE) {
1164       builder.setNonce(nonce);
1165     }
1166     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1167     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1168     for (Map.Entry<byte[],List<Cell>> family: mutation.getFamilyCellMap().entrySet()) {
1169       columnBuilder.clear();
1170       columnBuilder.setFamily(ByteStringer.wrap(family.getKey()));
1171       for (Cell cell: family.getValue()) {
1172         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
1173         valueBuilder.setQualifier(ByteStringer.wrap(
1174             kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()));
1175         valueBuilder.setValue(ByteStringer.wrap(
1176             kv.getValueArray(), kv.getValueOffset(), kv.getValueLength()));
1177         valueBuilder.setTimestamp(kv.getTimestamp());
1178         if(cell.getTagsLength() > 0) {
1179           valueBuilder.setTags(ByteStringer.wrap(kv.getTagsArray(), kv.getTagsOffset(),
1180               kv.getTagsLength()));
1181         }
1182         if (type == MutationType.DELETE || (type == MutationType.PUT && CellUtil.isDelete(kv))) {
1183           KeyValue.Type keyValueType = KeyValue.Type.codeToType(kv.getType());
1184           valueBuilder.setDeleteType(toDeleteType(keyValueType));
1185         }
1186         columnBuilder.addQualifierValue(valueBuilder.build());
1187       }
1188       builder.addColumnValue(columnBuilder.build());
1189     }
1190     return builder.build();
1191   }
1192 
1193   /**
1194    * Create a protocol buffer MutationProto based on a client Mutation. Does NOT include data.
1195    * Understanding is that the Cell will be transported other than via protobuf.
1196    * @param type
1197    * @param mutation
1198    * @param builder
1199    * @return a protobuf'd Mutation
1200    * @throws IOException
1201    */
1202   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1203       final MutationProto.Builder builder)  throws IOException {
1204     return toMutationNoData(type, mutation, builder, HConstants.NO_NONCE);
1205   }
1206 
1207   /**
1208    * Create a protocol buffer MutationProto based on a client Mutation.  Does NOT include data.
1209    * Understanding is that the Cell will be transported other than via protobuf.
1210    * @param type
1211    * @param mutation
1212    * @return a protobuf'd Mutation
1213    * @throws IOException
1214    */
1215   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation)
1216   throws IOException {
1217     MutationProto.Builder builder =  MutationProto.newBuilder();
1218     return toMutationNoData(type, mutation, builder);
1219   }
1220 
1221   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1222       final MutationProto.Builder builder, long nonce) throws IOException {
1223     getMutationBuilderAndSetCommonFields(type, mutation, builder);
1224     builder.setAssociatedCellCount(mutation.size());
1225     if (nonce != HConstants.NO_NONCE) {
1226       builder.setNonce(nonce);
1227     }
1228     return builder.build();
1229   }
1230 
1231   /**
1232    * Code shared by {@link #toMutation(MutationType, Mutation)} and
1233    * {@link #toMutationNoData(MutationType, Mutation)}
1234    * @param type
1235    * @param mutation
1236    * @return A partly-filled out protobuf'd Mutation.
1237    */
1238   private static MutationProto.Builder getMutationBuilderAndSetCommonFields(final MutationType type,
1239       final Mutation mutation, MutationProto.Builder builder) {
1240     builder.setRow(ByteStringer.wrap(mutation.getRow()));
1241     builder.setMutateType(type);
1242     builder.setDurability(toDurability(mutation.getDurability()));
1243     builder.setTimestamp(mutation.getTimeStamp());
1244     Map<String, byte[]> attributes = mutation.getAttributesMap();
1245     if (!attributes.isEmpty()) {
1246       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1247       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
1248         attributeBuilder.setName(attribute.getKey());
1249         attributeBuilder.setValue(ByteStringer.wrap(attribute.getValue()));
1250         builder.addAttribute(attributeBuilder.build());
1251       }
1252     }
1253     return builder;
1254   }
1255 
1256   /**
1257    * Convert a client Result to a protocol buffer Result
1258    *
1259    * @param result the client Result to convert
1260    * @return the converted protocol buffer Result
1261    */
1262   public static ClientProtos.Result toResult(final Result result) {
1263     if (result.getExists() != null) {
1264       return toResult(result.getExists(), result.isStale());
1265     }
1266 
1267     Cell[] cells = result.rawCells();
1268     if (cells == null || cells.length == 0) {
1269       return result.isStale() ? EMPTY_RESULT_PB_STALE : EMPTY_RESULT_PB;
1270     }
1271 
1272     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1273     for (Cell c : cells) {
1274       builder.addCell(toCell(c));
1275     }
1276 
1277     builder.setStale(result.isStale());
1278 
1279     return builder.build();
1280   }
1281 
1282   /**
1283    * Convert a client Result to a protocol buffer Result
1284    *
1285    * @param existence the client existence to send
1286    * @return the converted protocol buffer Result
1287    */
1288   public static ClientProtos.Result toResult(final boolean existence, boolean stale) {
1289     if (stale){
1290       return existence ? EMPTY_RESULT_PB_EXISTS_TRUE_STALE : EMPTY_RESULT_PB_EXISTS_FALSE_STALE;
1291     } else {
1292       return existence ? EMPTY_RESULT_PB_EXISTS_TRUE : EMPTY_RESULT_PB_EXISTS_FALSE;
1293     }
1294   }
1295 
1296   /**
1297    * Convert a client Result to a protocol buffer Result.
1298    * The pb Result does not include the Cell data.  That is for transport otherwise.
1299    *
1300    * @param result the client Result to convert
1301    * @return the converted protocol buffer Result
1302    */
1303   public static ClientProtos.Result toResultNoData(final Result result) {
1304     if (result.getExists() != null) return toResult(result.getExists(), result.isStale());
1305     int size = result.size();
1306     if (size == 0) return result.isStale() ? EMPTY_RESULT_PB_STALE : EMPTY_RESULT_PB;
1307     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1308     builder.setAssociatedCellCount(size);
1309     builder.setStale(result.isStale());
1310     return builder.build();
1311   }
1312 
1313   /**
1314    * Convert a protocol buffer Result to a client Result
1315    *
1316    * @param proto the protocol buffer Result to convert
1317    * @return the converted client Result
1318    */
1319   public static Result toResult(final ClientProtos.Result proto) {
1320     if (proto.hasExists()) {
1321       if (proto.getStale()) {
1322         return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE_STALE :EMPTY_RESULT_EXISTS_FALSE_STALE;
1323       }
1324       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1325     }
1326 
1327     List<CellProtos.Cell> values = proto.getCellList();
1328     if (values.isEmpty()){
1329       return proto.getStale() ? EMPTY_RESULT_STALE : EMPTY_RESULT;
1330     }
1331 
1332     List<Cell> cells = new ArrayList<Cell>(values.size());
1333     for (CellProtos.Cell c : values) {
1334       cells.add(toCell(c));
1335     }
1336     return Result.create(cells, null, proto.getStale());
1337   }
1338 
1339   /**
1340    * Convert a protocol buffer Result to a client Result
1341    *
1342    * @param proto the protocol buffer Result to convert
1343    * @param scanner Optional cell scanner.
1344    * @return the converted client Result
1345    * @throws IOException
1346    */
1347   public static Result toResult(final ClientProtos.Result proto, final CellScanner scanner)
1348   throws IOException {
1349     List<CellProtos.Cell> values = proto.getCellList();
1350 
1351     if (proto.hasExists()) {
1352       if ((values != null && !values.isEmpty()) ||
1353           (proto.hasAssociatedCellCount() && proto.getAssociatedCellCount() > 0)) {
1354         throw new IllegalArgumentException("bad proto: exists with cells is no allowed " + proto);
1355       }
1356       if (proto.getStale()) {
1357         return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE_STALE :EMPTY_RESULT_EXISTS_FALSE_STALE;
1358       }
1359       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1360     }
1361 
1362     // TODO: Unit test that has some Cells in scanner and some in the proto.
1363     List<Cell> cells = null;
1364     if (proto.hasAssociatedCellCount()) {
1365       int count = proto.getAssociatedCellCount();
1366       cells = new ArrayList<Cell>(count + values.size());
1367       for (int i = 0; i < count; i++) {
1368         if (!scanner.advance()) throw new IOException("Failed get " + i + " of " + count);
1369         cells.add(scanner.current());
1370       }
1371     }
1372 
1373     if (!values.isEmpty()){
1374       if (cells == null) cells = new ArrayList<Cell>(values.size());
1375       for (CellProtos.Cell c: values) {
1376         cells.add(toCell(c));
1377       }
1378     }
1379 
1380     return (cells == null || cells.isEmpty())
1381         ? (proto.getStale() ? EMPTY_RESULT_STALE : EMPTY_RESULT)
1382         : Result.create(cells, null, proto.getStale());
1383   }
1384 
1385 
1386   /**
1387    * Convert a ByteArrayComparable to a protocol buffer Comparator
1388    *
1389    * @param comparator the ByteArrayComparable to convert
1390    * @return the converted protocol buffer Comparator
1391    */
1392   public static ComparatorProtos.Comparator toComparator(ByteArrayComparable comparator) {
1393     ComparatorProtos.Comparator.Builder builder = ComparatorProtos.Comparator.newBuilder();
1394     builder.setName(comparator.getClass().getName());
1395     builder.setSerializedComparator(ByteStringer.wrap(comparator.toByteArray()));
1396     return builder.build();
1397   }
1398 
1399   /**
1400    * Convert a protocol buffer Comparator to a ByteArrayComparable
1401    *
1402    * @param proto the protocol buffer Comparator to convert
1403    * @return the converted ByteArrayComparable
1404    */
1405   @SuppressWarnings("unchecked")
1406   public static ByteArrayComparable toComparator(ComparatorProtos.Comparator proto)
1407   throws IOException {
1408     String type = proto.getName();
1409     String funcName = "parseFrom";
1410     byte [] value = proto.getSerializedComparator().toByteArray();
1411     try {
1412       Class<? extends ByteArrayComparable> c =
1413         (Class<? extends ByteArrayComparable>)Class.forName(type, true, CLASS_LOADER);
1414       Method parseFrom = c.getMethod(funcName, byte[].class);
1415       if (parseFrom == null) {
1416         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1417       }
1418       return (ByteArrayComparable)parseFrom.invoke(null, value);
1419     } catch (Exception e) {
1420       throw new IOException(e);
1421     }
1422   }
1423 
1424   /**
1425    * Convert a protocol buffer Filter to a client Filter
1426    *
1427    * @param proto the protocol buffer Filter to convert
1428    * @return the converted Filter
1429    */
1430   @SuppressWarnings("unchecked")
1431   public static Filter toFilter(FilterProtos.Filter proto) throws IOException {
1432     String type = proto.getName();
1433     final byte [] value = proto.getSerializedFilter().toByteArray();
1434     String funcName = "parseFrom";
1435     try {
1436       Class<? extends Filter> c =
1437         (Class<? extends Filter>)Class.forName(type, true, CLASS_LOADER);
1438       Method parseFrom = c.getMethod(funcName, byte[].class);
1439       if (parseFrom == null) {
1440         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1441       }
1442       return (Filter)parseFrom.invoke(c, value);
1443     } catch (Exception e) {
1444       throw new IOException(e);
1445     }
1446   }
1447 
1448   /**
1449    * Convert a client Filter to a protocol buffer Filter
1450    *
1451    * @param filter the Filter to convert
1452    * @return the converted protocol buffer Filter
1453    */
1454   public static FilterProtos.Filter toFilter(Filter filter) throws IOException {
1455     FilterProtos.Filter.Builder builder = FilterProtos.Filter.newBuilder();
1456     builder.setName(filter.getClass().getName());
1457     builder.setSerializedFilter(ByteStringer.wrap(filter.toByteArray()));
1458     return builder.build();
1459   }
1460 
1461   /**
1462    * Convert a delete KeyValue type to protocol buffer DeleteType.
1463    *
1464    * @param type
1465    * @return protocol buffer DeleteType
1466    * @throws IOException
1467    */
1468   public static DeleteType toDeleteType(
1469       KeyValue.Type type) throws IOException {
1470     switch (type) {
1471     case Delete:
1472       return DeleteType.DELETE_ONE_VERSION;
1473     case DeleteColumn:
1474       return DeleteType.DELETE_MULTIPLE_VERSIONS;
1475     case DeleteFamily:
1476       return DeleteType.DELETE_FAMILY;
1477     case DeleteFamilyVersion:
1478       return DeleteType.DELETE_FAMILY_VERSION;
1479     default:
1480         throw new IOException("Unknown delete type: " + type);
1481     }
1482   }
1483 
1484   /**
1485    * Convert a protocol buffer DeleteType to delete KeyValue type.
1486    *
1487    * @param type The DeleteType
1488    * @return The type.
1489    * @throws IOException
1490    */
1491   public static KeyValue.Type fromDeleteType(
1492       DeleteType type) throws IOException {
1493     switch (type) {
1494     case DELETE_ONE_VERSION:
1495       return KeyValue.Type.Delete;
1496     case DELETE_MULTIPLE_VERSIONS:
1497       return KeyValue.Type.DeleteColumn;
1498     case DELETE_FAMILY:
1499       return KeyValue.Type.DeleteFamily;
1500     case DELETE_FAMILY_VERSION:
1501       return KeyValue.Type.DeleteFamilyVersion;
1502     default:
1503       throw new IOException("Unknown delete type: " + type);
1504     }
1505   }
1506 
1507   /**
1508    * Convert a stringified protocol buffer exception Parameter to a Java Exception
1509    *
1510    * @param parameter the protocol buffer Parameter to convert
1511    * @return the converted Exception
1512    * @throws IOException if failed to deserialize the parameter
1513    */
1514   @SuppressWarnings("unchecked")
1515   public static Throwable toException(final NameBytesPair parameter) throws IOException {
1516     if (parameter == null || !parameter.hasValue()) return null;
1517     String desc = parameter.getValue().toStringUtf8();
1518     String type = parameter.getName();
1519     try {
1520       Class<? extends Throwable> c =
1521         (Class<? extends Throwable>)Class.forName(type, true, CLASS_LOADER);
1522       Constructor<? extends Throwable> cn = null;
1523       try {
1524         cn = c.getDeclaredConstructor(String.class);
1525         return cn.newInstance(desc);
1526       } catch (NoSuchMethodException e) {
1527         // Could be a raw RemoteException. See HBASE-8987.
1528         cn = c.getDeclaredConstructor(String.class, String.class);
1529         return cn.newInstance(type, desc);
1530       }
1531     } catch (Exception e) {
1532       throw new IOException(e);
1533     }
1534   }
1535 
1536 // Start helpers for Client
1537 
1538   /**
1539    * A helper to get a row of the closet one before using client protocol.
1540    *
1541    * @param client
1542    * @param regionName
1543    * @param row
1544    * @param family
1545    * @return the row or the closestRowBefore if it doesn't exist
1546    * @throws IOException
1547    * @deprecated since 0.99 - use reversed scanner instead.
1548    */
1549   @Deprecated
1550   public static Result getRowOrBefore(final ClientService.BlockingInterface client,
1551       final byte[] regionName, final byte[] row,
1552       final byte[] family) throws IOException {
1553     GetRequest request =
1554       RequestConverter.buildGetRowOrBeforeRequest(
1555         regionName, row, family);
1556     try {
1557       GetResponse response = client.get(null, request);
1558       if (!response.hasResult()) return null;
1559       return toResult(response.getResult());
1560     } catch (ServiceException se) {
1561       throw getRemoteException(se);
1562     }
1563   }
1564 
1565   /**
1566    * A helper to bulk load a list of HFiles using client protocol.
1567    *
1568    * @param client
1569    * @param familyPaths
1570    * @param regionName
1571    * @param assignSeqNum
1572    * @return true if all are loaded
1573    * @throws IOException
1574    */
1575   public static boolean bulkLoadHFile(final ClientService.BlockingInterface client,
1576       final List<Pair<byte[], String>> familyPaths,
1577       final byte[] regionName, boolean assignSeqNum) throws IOException {
1578     BulkLoadHFileRequest request =
1579       RequestConverter.buildBulkLoadHFileRequest(familyPaths, regionName, assignSeqNum);
1580     try {
1581       BulkLoadHFileResponse response =
1582         client.bulkLoadHFile(null, request);
1583       return response.getLoaded();
1584     } catch (ServiceException se) {
1585       throw getRemoteException(se);
1586     }
1587   }
1588 
1589   public static CoprocessorServiceResponse execService(final ClientService.BlockingInterface client,
1590       final CoprocessorServiceCall call, final byte[] regionName) throws IOException {
1591     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1592         .setCall(call).setRegion(
1593             RequestConverter.buildRegionSpecifier(REGION_NAME, regionName)).build();
1594     try {
1595       CoprocessorServiceResponse response =
1596           client.execService(null, request);
1597       return response;
1598     } catch (ServiceException se) {
1599       throw getRemoteException(se);
1600     }
1601   }
1602 
1603   public static CoprocessorServiceResponse execService(
1604     final MasterService.BlockingInterface client, final CoprocessorServiceCall call)
1605   throws IOException {
1606     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1607         .setCall(call).setRegion(
1608             RequestConverter.buildRegionSpecifier(REGION_NAME, HConstants.EMPTY_BYTE_ARRAY)).build();
1609     try {
1610       CoprocessorServiceResponse response =
1611           client.execMasterService(null, request);
1612       return response;
1613     } catch (ServiceException se) {
1614       throw getRemoteException(se);
1615     }
1616   }
1617 
1618   @SuppressWarnings("unchecked")
1619   public static <T extends Service> T newServiceStub(Class<T> service, RpcChannel channel)
1620       throws Exception {
1621     return (T)Methods.call(service, null, "newStub",
1622         new Class[]{ RpcChannel.class }, new Object[]{ channel });
1623   }
1624 
1625 // End helpers for Client
1626 // Start helpers for Admin
1627 
1628   /**
1629    * A helper to retrieve region info given a region name
1630    * using admin protocol.
1631    *
1632    * @param admin
1633    * @param regionName
1634    * @return the retrieved region info
1635    * @throws IOException
1636    */
1637   public static HRegionInfo getRegionInfo(final AdminService.BlockingInterface admin,
1638       final byte[] regionName) throws IOException {
1639     try {
1640       GetRegionInfoRequest request =
1641         RequestConverter.buildGetRegionInfoRequest(regionName);
1642       GetRegionInfoResponse response =
1643         admin.getRegionInfo(null, request);
1644       return HRegionInfo.convert(response.getRegionInfo());
1645     } catch (ServiceException se) {
1646       throw getRemoteException(se);
1647     }
1648   }
1649 
1650   /**
1651    * A helper to close a region given a region name
1652    * using admin protocol.
1653    *
1654    * @param admin
1655    * @param regionName
1656    * @throws IOException
1657    */
1658   public static void closeRegion(final AdminService.BlockingInterface admin,
1659       final ServerName server, final byte[] regionName) throws IOException {
1660     CloseRegionRequest closeRegionRequest =
1661       RequestConverter.buildCloseRegionRequest(server, regionName);
1662     try {
1663       admin.closeRegion(null, closeRegionRequest);
1664     } catch (ServiceException se) {
1665       throw getRemoteException(se);
1666     }
1667   }
1668 
1669   /**
1670    * A helper to close a region given a region name
1671    * using admin protocol.
1672    *
1673    * @param admin
1674    * @param regionName
1675    * @return true if the region is closed
1676    * @throws IOException
1677    */
1678   public static boolean closeRegion(final AdminService.BlockingInterface admin,
1679       final ServerName server, final byte[] regionName,
1680       final ServerName destinationServer) throws IOException {
1681     CloseRegionRequest closeRegionRequest =
1682       RequestConverter.buildCloseRegionRequest(server,
1683         regionName, destinationServer);
1684     try {
1685       CloseRegionResponse response = admin.closeRegion(null, closeRegionRequest);
1686       return ResponseConverter.isClosed(response);
1687     } catch (ServiceException se) {
1688       throw getRemoteException(se);
1689     }
1690   }
1691 
1692 
1693   /**
1694    * A helper to open a region using admin protocol.
1695    * @param admin
1696    * @param region
1697    * @throws IOException
1698    */
1699   public static void openRegion(final AdminService.BlockingInterface admin,
1700       ServerName server, final HRegionInfo region) throws IOException {
1701     OpenRegionRequest request =
1702       RequestConverter.buildOpenRegionRequest(server, region, null, null);
1703     try {
1704       admin.openRegion(null, request);
1705     } catch (ServiceException se) {
1706       throw ProtobufUtil.getRemoteException(se);
1707     }
1708   }
1709 
1710   /**
1711    * A helper to get the all the online regions on a region
1712    * server using admin protocol.
1713    *
1714    * @param admin
1715    * @return a list of online region info
1716    * @throws IOException
1717    */
1718   public static List<HRegionInfo> getOnlineRegions(final AdminService.BlockingInterface admin)
1719   throws IOException {
1720     GetOnlineRegionRequest request = RequestConverter.buildGetOnlineRegionRequest();
1721     GetOnlineRegionResponse response = null;
1722     try {
1723       response = admin.getOnlineRegion(null, request);
1724     } catch (ServiceException se) {
1725       throw getRemoteException(se);
1726     }
1727     return getRegionInfos(response);
1728   }
1729 
1730   /**
1731    * Get the list of region info from a GetOnlineRegionResponse
1732    *
1733    * @param proto the GetOnlineRegionResponse
1734    * @return the list of region info or null if <code>proto</code> is null
1735    */
1736   static List<HRegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
1737     if (proto == null) return null;
1738     List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
1739     for (RegionInfo regionInfo: proto.getRegionInfoList()) {
1740       regionInfos.add(HRegionInfo.convert(regionInfo));
1741     }
1742     return regionInfos;
1743   }
1744 
1745   /**
1746    * A helper to get the info of a region server using admin protocol.
1747    *
1748    * @param admin
1749    * @return the server name
1750    * @throws IOException
1751    */
1752   public static ServerInfo getServerInfo(final AdminService.BlockingInterface admin)
1753   throws IOException {
1754     GetServerInfoRequest request = RequestConverter.buildGetServerInfoRequest();
1755     try {
1756       GetServerInfoResponse response = admin.getServerInfo(null, request);
1757       return response.getServerInfo();
1758     } catch (ServiceException se) {
1759       throw getRemoteException(se);
1760     }
1761   }
1762 
1763   /**
1764    * A helper to get the list of files of a column family
1765    * on a given region using admin protocol.
1766    *
1767    * @param admin
1768    * @param regionName
1769    * @param family
1770    * @return the list of store files
1771    * @throws IOException
1772    */
1773   public static List<String> getStoreFiles(final AdminService.BlockingInterface admin,
1774       final byte[] regionName, final byte[] family)
1775   throws IOException {
1776     GetStoreFileRequest request =
1777       RequestConverter.buildGetStoreFileRequest(regionName, family);
1778     try {
1779       GetStoreFileResponse response = admin.getStoreFile(null, request);
1780       return response.getStoreFileList();
1781     } catch (ServiceException se) {
1782       throw ProtobufUtil.getRemoteException(se);
1783     }
1784   }
1785 
1786   /**
1787    * A helper to split a region using admin protocol.
1788    *
1789    * @param admin
1790    * @param hri
1791    * @param splitPoint
1792    * @throws IOException
1793    */
1794   public static void split(final AdminService.BlockingInterface admin,
1795       final HRegionInfo hri, byte[] splitPoint) throws IOException {
1796     SplitRegionRequest request =
1797       RequestConverter.buildSplitRegionRequest(hri.getRegionName(), splitPoint);
1798     try {
1799       admin.splitRegion(null, request);
1800     } catch (ServiceException se) {
1801       throw ProtobufUtil.getRemoteException(se);
1802     }
1803   }
1804 
1805   /**
1806    * A helper to merge regions using admin protocol. Send request to
1807    * regionserver.
1808    * @param admin
1809    * @param region_a
1810    * @param region_b
1811    * @param forcible true if do a compulsory merge, otherwise we will only merge
1812    *          two adjacent regions
1813    * @throws IOException
1814    */
1815   public static void mergeRegions(final AdminService.BlockingInterface admin,
1816       final HRegionInfo region_a, final HRegionInfo region_b,
1817       final boolean forcible) throws IOException {
1818     MergeRegionsRequest request = RequestConverter.buildMergeRegionsRequest(
1819         region_a.getRegionName(), region_b.getRegionName(),forcible);
1820     try {
1821       admin.mergeRegions(null, request);
1822     } catch (ServiceException se) {
1823       throw ProtobufUtil.getRemoteException(se);
1824     }
1825   }
1826 
1827 // End helpers for Admin
1828 
1829   /*
1830    * Get the total (read + write) requests from a RegionLoad pb
1831    * @param rl - RegionLoad pb
1832    * @return total (read + write) requests
1833    */
1834   public static long getTotalRequestsCount(RegionLoad rl) {
1835     if (rl == null) {
1836       return 0;
1837     }
1838 
1839     return rl.getReadRequestsCount() + rl.getWriteRequestsCount();
1840   }
1841 
1842 
1843   /**
1844    * @param m Message to get delimited pb serialization of (with pb magic prefix)
1845    */
1846   public static byte [] toDelimitedByteArray(final Message m) throws IOException {
1847     // Allocate arbitrary big size so we avoid resizing.
1848     ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
1849     baos.write(PB_MAGIC);
1850     m.writeDelimitedTo(baos);
1851     return baos.toByteArray();
1852   }
1853 
1854   /**
1855    * Converts a Permission proto to a client Permission object.
1856    *
1857    * @param proto the protobuf Permission
1858    * @return the converted Permission
1859    */
1860   public static Permission toPermission(AccessControlProtos.Permission proto) {
1861     if (proto.getType() != AccessControlProtos.Permission.Type.Global) {
1862       return toTablePermission(proto);
1863     } else {
1864       List<Permission.Action> actions = toPermissionActions(proto.getGlobalPermission().getActionList());
1865       return new Permission(actions.toArray(new Permission.Action[actions.size()]));
1866     }
1867   }
1868 
1869   /**
1870    * Converts a Permission proto to a client TablePermission object.
1871    *
1872    * @param proto the protobuf Permission
1873    * @return the converted TablePermission
1874    */
1875   public static TablePermission toTablePermission(AccessControlProtos.Permission proto) {
1876     if(proto.getType() == AccessControlProtos.Permission.Type.Global) {
1877       AccessControlProtos.GlobalPermission perm = proto.getGlobalPermission();
1878       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1879 
1880       return new TablePermission(null, null, null,
1881           actions.toArray(new Permission.Action[actions.size()]));
1882     }
1883     if(proto.getType() == AccessControlProtos.Permission.Type.Namespace) {
1884       AccessControlProtos.NamespacePermission perm = proto.getNamespacePermission();
1885       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1886 
1887       if(!proto.hasNamespacePermission()) {
1888         throw new IllegalStateException("Namespace must not be empty in NamespacePermission");
1889       }
1890       String namespace = perm.getNamespaceName().toStringUtf8();
1891       return new TablePermission(namespace, actions.toArray(new Permission.Action[actions.size()]));
1892     }
1893     if(proto.getType() == AccessControlProtos.Permission.Type.Table) {
1894       AccessControlProtos.TablePermission perm = proto.getTablePermission();
1895       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1896 
1897       byte[] qualifier = null;
1898       byte[] family = null;
1899       TableName table = null;
1900 
1901       if (!perm.hasTableName()) {
1902         throw new IllegalStateException("TableName cannot be empty");
1903       }
1904       table = ProtobufUtil.toTableName(perm.getTableName());
1905 
1906       if (perm.hasFamily()) family = perm.getFamily().toByteArray();
1907       if (perm.hasQualifier()) qualifier = perm.getQualifier().toByteArray();
1908 
1909       return new TablePermission(table, family, qualifier,
1910           actions.toArray(new Permission.Action[actions.size()]));
1911     }
1912     throw new IllegalStateException("Unrecognize Perm Type: "+proto.getType());
1913   }
1914 
1915   /**
1916    * Convert a client Permission to a Permission proto
1917    *
1918    * @param perm the client Permission
1919    * @return the protobuf Permission
1920    */
1921   public static AccessControlProtos.Permission toPermission(Permission perm) {
1922     AccessControlProtos.Permission.Builder ret = AccessControlProtos.Permission.newBuilder();
1923     if (perm instanceof TablePermission) {
1924       TablePermission tablePerm = (TablePermission)perm;
1925       if(tablePerm.hasNamespace()) {
1926         ret.setType(AccessControlProtos.Permission.Type.Namespace);
1927 
1928         AccessControlProtos.NamespacePermission.Builder builder =
1929             AccessControlProtos.NamespacePermission.newBuilder();
1930         builder.setNamespaceName(ByteString.copyFromUtf8(tablePerm.getNamespace()));
1931         Permission.Action actions[] = perm.getActions();
1932         if (actions != null) {
1933           for (Permission.Action a : actions) {
1934             builder.addAction(toPermissionAction(a));
1935           }
1936 	}
1937         ret.setNamespacePermission(builder);
1938         return ret.build();
1939       } else if (tablePerm.hasTable()) {
1940         ret.setType(AccessControlProtos.Permission.Type.Table);
1941 
1942         AccessControlProtos.TablePermission.Builder builder =
1943             AccessControlProtos.TablePermission.newBuilder();
1944         builder.setTableName(ProtobufUtil.toProtoTableName(tablePerm.getTableName()));
1945         if (tablePerm.hasFamily()) {
1946           builder.setFamily(ByteStringer.wrap(tablePerm.getFamily()));
1947         }
1948         if (tablePerm.hasQualifier()) {
1949           builder.setQualifier(ByteStringer.wrap(tablePerm.getQualifier()));
1950         }
1951         Permission.Action actions[] = perm.getActions();
1952         if (actions != null) {
1953           for (Permission.Action a : actions) {
1954             builder.addAction(toPermissionAction(a));
1955           }
1956         }
1957         ret.setTablePermission(builder);
1958         return ret.build();
1959       }
1960     }
1961 
1962     ret.setType(AccessControlProtos.Permission.Type.Global);
1963 
1964     AccessControlProtos.GlobalPermission.Builder builder =
1965         AccessControlProtos.GlobalPermission.newBuilder();
1966     Permission.Action actions[] = perm.getActions();
1967     if (actions != null) {
1968       for (Permission.Action a: actions) {
1969         builder.addAction(toPermissionAction(a));
1970       }
1971     }
1972     ret.setGlobalPermission(builder);
1973     return ret.build();
1974   }
1975 
1976   /**
1977    * Converts a list of Permission.Action proto to a list of client Permission.Action objects.
1978    *
1979    * @param protoActions the list of protobuf Actions
1980    * @return the converted list of Actions
1981    */
1982   public static List<Permission.Action> toPermissionActions(
1983       List<AccessControlProtos.Permission.Action> protoActions) {
1984     List<Permission.Action> actions = new ArrayList<Permission.Action>(protoActions.size());
1985     for (AccessControlProtos.Permission.Action a : protoActions) {
1986       actions.add(toPermissionAction(a));
1987     }
1988     return actions;
1989   }
1990 
1991   /**
1992    * Converts a Permission.Action proto to a client Permission.Action object.
1993    *
1994    * @param action the protobuf Action
1995    * @return the converted Action
1996    */
1997   public static Permission.Action toPermissionAction(
1998       AccessControlProtos.Permission.Action action) {
1999     switch (action) {
2000       case READ:
2001         return Permission.Action.READ;
2002       case WRITE:
2003         return Permission.Action.WRITE;
2004       case EXEC:
2005         return Permission.Action.EXEC;
2006       case CREATE:
2007         return Permission.Action.CREATE;
2008       case ADMIN:
2009         return Permission.Action.ADMIN;
2010     }
2011     throw new IllegalArgumentException("Unknown action value "+action.name());
2012   }
2013 
2014   /**
2015    * Convert a client Permission.Action to a Permission.Action proto
2016    *
2017    * @param action the client Action
2018    * @return the protobuf Action
2019    */
2020   public static AccessControlProtos.Permission.Action toPermissionAction(
2021       Permission.Action action) {
2022     switch (action) {
2023       case READ:
2024         return AccessControlProtos.Permission.Action.READ;
2025       case WRITE:
2026         return AccessControlProtos.Permission.Action.WRITE;
2027       case EXEC:
2028         return AccessControlProtos.Permission.Action.EXEC;
2029       case CREATE:
2030         return AccessControlProtos.Permission.Action.CREATE;
2031       case ADMIN:
2032         return AccessControlProtos.Permission.Action.ADMIN;
2033     }
2034     throw new IllegalArgumentException("Unknown action value "+action.name());
2035   }
2036 
2037   /**
2038    * Convert a client user permission to a user permission proto
2039    *
2040    * @param perm the client UserPermission
2041    * @return the protobuf UserPermission
2042    */
2043   public static AccessControlProtos.UserPermission toUserPermission(UserPermission perm) {
2044     return AccessControlProtos.UserPermission.newBuilder()
2045         .setUser(ByteStringer.wrap(perm.getUser()))
2046         .setPermission(toPermission(perm))
2047         .build();
2048   }
2049 
2050   /**
2051    * Converts a user permission proto to a client user permission object.
2052    *
2053    * @param proto the protobuf UserPermission
2054    * @return the converted UserPermission
2055    */
2056   public static UserPermission toUserPermission(AccessControlProtos.UserPermission proto) {
2057     return new UserPermission(proto.getUser().toByteArray(),
2058         toTablePermission(proto.getPermission()));
2059   }
2060 
2061   /**
2062    * Convert a ListMultimap<String, TablePermission> where key is username
2063    * to a protobuf UserPermission
2064    *
2065    * @param perm the list of user and table permissions
2066    * @return the protobuf UserTablePermissions
2067    */
2068   public static AccessControlProtos.UsersAndPermissions toUserTablePermissions(
2069       ListMultimap<String, TablePermission> perm) {
2070     AccessControlProtos.UsersAndPermissions.Builder builder =
2071                   AccessControlProtos.UsersAndPermissions.newBuilder();
2072     for (Map.Entry<String, Collection<TablePermission>> entry : perm.asMap().entrySet()) {
2073       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
2074                   AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
2075       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
2076       for (TablePermission tablePerm: entry.getValue()) {
2077         userPermBuilder.addPermissions(toPermission(tablePerm));
2078       }
2079       builder.addUserPermissions(userPermBuilder.build());
2080     }
2081     return builder.build();
2082   }
2083 
2084   /**
2085    * A utility used to grant a user global permissions.
2086    * <p>
2087    * It's also called by the shell, in case you want to find references.
2088    *
2089    * @param protocol the AccessControlService protocol proxy
2090    * @param userShortName the short name of the user to grant permissions
2091    * @param actions the permissions to be granted
2092    * @throws ServiceException
2093    */
2094   public static void grant(AccessControlService.BlockingInterface protocol,
2095       String userShortName, Permission.Action... actions) throws ServiceException {
2096     List<AccessControlProtos.Permission.Action> permActions =
2097         Lists.newArrayListWithCapacity(actions.length);
2098     for (Permission.Action a : actions) {
2099       permActions.add(ProtobufUtil.toPermissionAction(a));
2100     }
2101     AccessControlProtos.GrantRequest request = RequestConverter.
2102       buildGrantRequest(userShortName, permActions.toArray(
2103         new AccessControlProtos.Permission.Action[actions.length]));
2104     protocol.grant(null, request);
2105   }
2106 
2107   /**
2108    * A utility used to grant a user table permissions. The permissions will
2109    * be for a table table/column family/qualifier.
2110    * <p>
2111    * It's also called by the shell, in case you want to find references.
2112    *
2113    * @param protocol the AccessControlService protocol proxy
2114    * @param userShortName the short name of the user to grant permissions
2115    * @param tableName optional table name
2116    * @param f optional column family
2117    * @param q optional qualifier
2118    * @param actions the permissions to be granted
2119    * @throws ServiceException
2120    */
2121   public static void grant(AccessControlService.BlockingInterface protocol,
2122       String userShortName, TableName tableName, byte[] f, byte[] q,
2123       Permission.Action... actions) throws ServiceException {
2124     List<AccessControlProtos.Permission.Action> permActions =
2125         Lists.newArrayListWithCapacity(actions.length);
2126     for (Permission.Action a : actions) {
2127       permActions.add(ProtobufUtil.toPermissionAction(a));
2128     }
2129     AccessControlProtos.GrantRequest request = RequestConverter.
2130       buildGrantRequest(userShortName, tableName, f, q, permActions.toArray(
2131         new AccessControlProtos.Permission.Action[actions.length]));
2132     protocol.grant(null, request);
2133   }
2134 
2135   /**
2136    * A utility used to grant a user namespace permissions.
2137    * <p>
2138    * It's also called by the shell, in case you want to find references.
2139    *
2140    * @param protocol the AccessControlService protocol proxy
2141    * @param namespace the short name of the user to grant permissions
2142    * @param actions the permissions to be granted
2143    * @throws ServiceException
2144    */
2145   public static void grant(AccessControlService.BlockingInterface protocol,
2146       String userShortName, String namespace,
2147       Permission.Action... actions) throws ServiceException {
2148     List<AccessControlProtos.Permission.Action> permActions =
2149         Lists.newArrayListWithCapacity(actions.length);
2150     for (Permission.Action a : actions) {
2151       permActions.add(ProtobufUtil.toPermissionAction(a));
2152     }
2153     AccessControlProtos.GrantRequest request = RequestConverter.
2154       buildGrantRequest(userShortName, namespace, permActions.toArray(
2155         new AccessControlProtos.Permission.Action[actions.length]));
2156     protocol.grant(null, request);
2157   }
2158 
2159   /**
2160    * A utility used to revoke a user's global permissions.
2161    * <p>
2162    * It's also called by the shell, in case you want to find references.
2163    *
2164    * @param protocol the AccessControlService protocol proxy
2165    * @param userShortName the short name of the user to revoke permissions
2166    * @param actions the permissions to be revoked
2167    * @throws ServiceException
2168    */
2169   public static void revoke(AccessControlService.BlockingInterface protocol,
2170       String userShortName, Permission.Action... actions) throws ServiceException {
2171     List<AccessControlProtos.Permission.Action> permActions =
2172         Lists.newArrayListWithCapacity(actions.length);
2173     for (Permission.Action a : actions) {
2174       permActions.add(ProtobufUtil.toPermissionAction(a));
2175     }
2176     AccessControlProtos.RevokeRequest request = RequestConverter.
2177       buildRevokeRequest(userShortName, permActions.toArray(
2178         new AccessControlProtos.Permission.Action[actions.length]));
2179     protocol.revoke(null, request);
2180   }
2181 
2182   /**
2183    * A utility used to revoke a user's table permissions. The permissions will
2184    * be for a table/column family/qualifier.
2185    * <p>
2186    * It's also called by the shell, in case you want to find references.
2187    *
2188    * @param protocol the AccessControlService protocol proxy
2189    * @param userShortName the short name of the user to revoke permissions
2190    * @param tableName optional table name
2191    * @param f optional column family
2192    * @param q optional qualifier
2193    * @param actions the permissions to be revoked
2194    * @throws ServiceException
2195    */
2196   public static void revoke(AccessControlService.BlockingInterface protocol,
2197       String userShortName, TableName tableName, byte[] f, byte[] q,
2198       Permission.Action... actions) throws ServiceException {
2199     List<AccessControlProtos.Permission.Action> permActions =
2200         Lists.newArrayListWithCapacity(actions.length);
2201     for (Permission.Action a : actions) {
2202       permActions.add(ProtobufUtil.toPermissionAction(a));
2203     }
2204     AccessControlProtos.RevokeRequest request = RequestConverter.
2205       buildRevokeRequest(userShortName, tableName, f, q, permActions.toArray(
2206         new AccessControlProtos.Permission.Action[actions.length]));
2207     protocol.revoke(null, request);
2208   }
2209 
2210   /**
2211    * A utility used to revoke a user's namespace permissions.
2212    * <p>
2213    * It's also called by the shell, in case you want to find references.
2214    *
2215    * @param protocol the AccessControlService protocol proxy
2216    * @param userShortName the short name of the user to revoke permissions
2217    * @param namespace optional table name
2218    * @param actions the permissions to be revoked
2219    * @throws ServiceException
2220    */
2221   public static void revoke(AccessControlService.BlockingInterface protocol,
2222       String userShortName, String namespace,
2223       Permission.Action... actions) throws ServiceException {
2224     List<AccessControlProtos.Permission.Action> permActions =
2225         Lists.newArrayListWithCapacity(actions.length);
2226     for (Permission.Action a : actions) {
2227       permActions.add(ProtobufUtil.toPermissionAction(a));
2228     }
2229     AccessControlProtos.RevokeRequest request = RequestConverter.
2230       buildRevokeRequest(userShortName, namespace, permActions.toArray(
2231         new AccessControlProtos.Permission.Action[actions.length]));
2232     protocol.revoke(null, request);
2233   }
2234 
2235   /**
2236    * A utility used to get user's global permissions.
2237    * <p>
2238    * It's also called by the shell, in case you want to find references.
2239    *
2240    * @param protocol the AccessControlService protocol proxy
2241    * @throws ServiceException
2242    */
2243   public static List<UserPermission> getUserPermissions(
2244       AccessControlService.BlockingInterface protocol) throws ServiceException {
2245     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2246       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2247     builder.setType(AccessControlProtos.Permission.Type.Global);
2248     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2249     AccessControlProtos.GetUserPermissionsResponse response =
2250       protocol.getUserPermissions(null, request);
2251     List<UserPermission> perms = new ArrayList<UserPermission>();
2252     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2253       perms.add(ProtobufUtil.toUserPermission(perm));
2254     }
2255     return perms;
2256   }
2257 
2258   /**
2259    * A utility used to get user table permissions.
2260    * <p>
2261    * It's also called by the shell, in case you want to find references.
2262    *
2263    * @param protocol the AccessControlService protocol proxy
2264    * @param t optional table name
2265    * @throws ServiceException
2266    */
2267   public static List<UserPermission> getUserPermissions(
2268       AccessControlService.BlockingInterface protocol,
2269       TableName t) throws ServiceException {
2270     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2271       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2272     if (t != null) {
2273       builder.setTableName(ProtobufUtil.toProtoTableName(t));
2274     }
2275     builder.setType(AccessControlProtos.Permission.Type.Table);
2276     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2277     AccessControlProtos.GetUserPermissionsResponse response =
2278       protocol.getUserPermissions(null, request);
2279     List<UserPermission> perms = new ArrayList<UserPermission>();
2280     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2281       perms.add(ProtobufUtil.toUserPermission(perm));
2282     }
2283     return perms;
2284   }
2285 
2286   /**
2287    * A utility used to get permissions for selected namespace.
2288    * <p>
2289    * It's also called by the shell, in case you want to find references.
2290    *
2291    * @param protocol the AccessControlService protocol proxy
2292    * @param namespace name of the namespace
2293    * @throws ServiceException
2294    */
2295   public static List<UserPermission> getUserPermissions(
2296       AccessControlService.BlockingInterface protocol,
2297       byte[] namespace) throws ServiceException {
2298     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2299       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2300     if (namespace != null) {
2301       builder.setNamespaceName(ByteStringer.wrap(namespace));
2302     }
2303     builder.setType(AccessControlProtos.Permission.Type.Namespace);
2304     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2305     AccessControlProtos.GetUserPermissionsResponse response =
2306       protocol.getUserPermissions(null, request);
2307     List<UserPermission> perms = new ArrayList<UserPermission>();
2308     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2309       perms.add(ProtobufUtil.toUserPermission(perm));
2310     }
2311     return perms;
2312   }
2313 
2314   /**
2315    * Convert a protobuf UserTablePermissions to a
2316    * ListMultimap<String, TablePermission> where key is username.
2317    *
2318    * @param proto the protobuf UserPermission
2319    * @return the converted UserPermission
2320    */
2321   public static ListMultimap<String, TablePermission> toUserTablePermissions(
2322       AccessControlProtos.UsersAndPermissions proto) {
2323     ListMultimap<String, TablePermission> perms = ArrayListMultimap.create();
2324     AccessControlProtos.UsersAndPermissions.UserPermissions userPerm;
2325 
2326     for (int i = 0; i < proto.getUserPermissionsCount(); i++) {
2327       userPerm = proto.getUserPermissions(i);
2328       for (int j = 0; j < userPerm.getPermissionsCount(); j++) {
2329         TablePermission tablePerm = toTablePermission(userPerm.getPermissions(j));
2330         perms.put(userPerm.getUser().toStringUtf8(), tablePerm);
2331       }
2332     }
2333 
2334     return perms;
2335   }
2336 
2337   /**
2338    * Converts a Token instance (with embedded identifier) to the protobuf representation.
2339    *
2340    * @param token the Token instance to copy
2341    * @return the protobuf Token message
2342    */
2343   public static AuthenticationProtos.Token toToken(Token<AuthenticationTokenIdentifier> token) {
2344     AuthenticationProtos.Token.Builder builder = AuthenticationProtos.Token.newBuilder();
2345     builder.setIdentifier(ByteStringer.wrap(token.getIdentifier()));
2346     builder.setPassword(ByteStringer.wrap(token.getPassword()));
2347     if (token.getService() != null) {
2348       builder.setService(ByteString.copyFromUtf8(token.getService().toString()));
2349     }
2350     return builder.build();
2351   }
2352 
2353   /**
2354    * Converts a protobuf Token message back into a Token instance.
2355    *
2356    * @param proto the protobuf Token message
2357    * @return the Token instance
2358    */
2359   public static Token<AuthenticationTokenIdentifier> toToken(AuthenticationProtos.Token proto) {
2360     return new Token<AuthenticationTokenIdentifier>(
2361         proto.hasIdentifier() ? proto.getIdentifier().toByteArray() : null,
2362         proto.hasPassword() ? proto.getPassword().toByteArray() : null,
2363         AuthenticationTokenIdentifier.AUTH_TOKEN_TYPE,
2364         proto.hasService() ? new Text(proto.getService().toStringUtf8()) : null);
2365   }
2366 
2367   /**
2368    * Find the HRegion encoded name based on a region specifier
2369    *
2370    * @param regionSpecifier the region specifier
2371    * @return the corresponding region's encoded name
2372    * @throws DoNotRetryIOException if the specifier type is unsupported
2373    */
2374   public static String getRegionEncodedName(
2375       final RegionSpecifier regionSpecifier) throws DoNotRetryIOException {
2376     byte[] value = regionSpecifier.getValue().toByteArray();
2377     RegionSpecifierType type = regionSpecifier.getType();
2378     switch (type) {
2379       case REGION_NAME:
2380         return HRegionInfo.encodeRegionName(value);
2381       case ENCODED_REGION_NAME:
2382         return Bytes.toString(value);
2383       default:
2384         throw new DoNotRetryIOException(
2385           "Unsupported region specifier type: " + type);
2386     }
2387   }
2388 
2389   public static ScanMetrics toScanMetrics(final byte[] bytes) {
2390     Parser<MapReduceProtos.ScanMetrics> parser = MapReduceProtos.ScanMetrics.PARSER;
2391     MapReduceProtos.ScanMetrics pScanMetrics = null;
2392     try {
2393       pScanMetrics = parser.parseFrom(bytes);
2394     } catch (InvalidProtocolBufferException e) {
2395       //Ignored there are just no key values to add.
2396     }
2397     ScanMetrics scanMetrics = new ScanMetrics();
2398     if (pScanMetrics != null) {
2399       for (HBaseProtos.NameInt64Pair pair : pScanMetrics.getMetricsList()) {
2400         if (pair.hasName() && pair.hasValue()) {
2401           scanMetrics.setCounter(pair.getName(), pair.getValue());
2402         }
2403       }
2404     }
2405     return scanMetrics;
2406   }
2407 
2408   public static MapReduceProtos.ScanMetrics toScanMetrics(ScanMetrics scanMetrics) {
2409     MapReduceProtos.ScanMetrics.Builder builder = MapReduceProtos.ScanMetrics.newBuilder();
2410     Map<String, Long> metrics = scanMetrics.getMetricsMap();
2411     for (Entry<String, Long> e : metrics.entrySet()) {
2412       HBaseProtos.NameInt64Pair nameInt64Pair =
2413           HBaseProtos.NameInt64Pair.newBuilder()
2414               .setName(e.getKey())
2415               .setValue(e.getValue())
2416               .build();
2417       builder.addMetrics(nameInt64Pair);
2418     }
2419     return builder.build();
2420   }
2421 
2422   /**
2423    * Unwraps an exception from a protobuf service into the underlying (expected) IOException.
2424    * This method will <strong>always</strong> throw an exception.
2425    * @param se the {@code ServiceException} instance to convert into an {@code IOException}
2426    */
2427   public static void toIOException(ServiceException se) throws IOException {
2428     if (se == null) {
2429       throw new NullPointerException("Null service exception passed!");
2430     }
2431 
2432     Throwable cause = se.getCause();
2433     if (cause != null && cause instanceof IOException) {
2434       throw (IOException)cause;
2435     }
2436     throw new IOException(se);
2437   }
2438 
2439   public static CellProtos.Cell toCell(final Cell kv) {
2440     // Doing this is going to kill us if we do it for all data passed.
2441     // St.Ack 20121205
2442     CellProtos.Cell.Builder kvbuilder = CellProtos.Cell.newBuilder();
2443     kvbuilder.setRow(ByteStringer.wrap(kv.getRowArray(), kv.getRowOffset(),
2444         kv.getRowLength()));
2445     kvbuilder.setFamily(ByteStringer.wrap(kv.getFamilyArray(),
2446         kv.getFamilyOffset(), kv.getFamilyLength()));
2447     kvbuilder.setQualifier(ByteStringer.wrap(kv.getQualifierArray(),
2448         kv.getQualifierOffset(), kv.getQualifierLength()));
2449     kvbuilder.setCellType(CellProtos.CellType.valueOf(kv.getTypeByte()));
2450     kvbuilder.setTimestamp(kv.getTimestamp());
2451     kvbuilder.setValue(ByteStringer.wrap(kv.getValueArray(), kv.getValueOffset(),
2452         kv.getValueLength()));
2453     return kvbuilder.build();
2454   }
2455 
2456   public static Cell toCell(final CellProtos.Cell cell) {
2457     // Doing this is going to kill us if we do it for all data passed.
2458     // St.Ack 20121205
2459     return CellUtil.createCell(cell.getRow().toByteArray(),
2460       cell.getFamily().toByteArray(),
2461       cell.getQualifier().toByteArray(),
2462       cell.getTimestamp(),
2463       (byte)cell.getCellType().getNumber(),
2464       cell.getValue().toByteArray());
2465   }
2466 
2467   public static HBaseProtos.NamespaceDescriptor toProtoNamespaceDescriptor(NamespaceDescriptor ns) {
2468     HBaseProtos.NamespaceDescriptor.Builder b =
2469         HBaseProtos.NamespaceDescriptor.newBuilder()
2470             .setName(ByteString.copyFromUtf8(ns.getName()));
2471     for(Map.Entry<String, String> entry: ns.getConfiguration().entrySet()) {
2472       b.addConfiguration(HBaseProtos.NameStringPair.newBuilder()
2473           .setName(entry.getKey())
2474           .setValue(entry.getValue()));
2475     }
2476     return b.build();
2477   }
2478 
2479   public static NamespaceDescriptor toNamespaceDescriptor(
2480       HBaseProtos.NamespaceDescriptor desc) throws IOException {
2481     NamespaceDescriptor.Builder b =
2482       NamespaceDescriptor.create(desc.getName().toStringUtf8());
2483     for(HBaseProtos.NameStringPair prop : desc.getConfigurationList()) {
2484       b.addConfiguration(prop.getName(), prop.getValue());
2485     }
2486     return b.build();
2487   }
2488 
2489   /**
2490    * Get an instance of the argument type declared in a class's signature. The
2491    * argument type is assumed to be a PB Message subclass, and the instance is
2492    * created using parseFrom method on the passed ByteString.
2493    * @param runtimeClass the runtime type of the class
2494    * @param position the position of the argument in the class declaration
2495    * @param b the ByteString which should be parsed to get the instance created
2496    * @return the instance
2497    * @throws IOException
2498    */
2499   @SuppressWarnings("unchecked")
2500   public static <T extends Message>
2501   T getParsedGenericInstance(Class<?> runtimeClass, int position, ByteString b)
2502       throws IOException {
2503     Type type = runtimeClass.getGenericSuperclass();
2504     Type argType = ((ParameterizedType)type).getActualTypeArguments()[position];
2505     Class<T> classType = (Class<T>)argType;
2506     T inst;
2507     try {
2508       Method m = classType.getMethod("parseFrom", ByteString.class);
2509       inst = (T)m.invoke(null, b);
2510       return inst;
2511     } catch (SecurityException e) {
2512       throw new IOException(e);
2513     } catch (NoSuchMethodException e) {
2514       throw new IOException(e);
2515     } catch (IllegalArgumentException e) {
2516       throw new IOException(e);
2517     } catch (InvocationTargetException e) {
2518       throw new IOException(e);
2519     } catch (IllegalAccessException e) {
2520       throw new IOException(e);
2521     }
2522   }
2523 
2524   @SuppressWarnings("deprecation")
2525   public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family,
2526       List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
2527     // compaction descriptor contains relative paths.
2528     // input / output paths are relative to the store dir
2529     // store dir is relative to region dir
2530     CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder()
2531         .setTableName(ByteStringer.wrap(info.getTableName()))
2532         .setEncodedRegionName(ByteStringer.wrap(info.getEncodedNameAsBytes()))
2533         .setFamilyName(ByteStringer.wrap(family))
2534         .setStoreHomeDir(storeDir.getName()); //make relative
2535     for (Path inputPath : inputPaths) {
2536       builder.addCompactionInput(inputPath.getName()); //relative path
2537     }
2538     for (Path outputPath : outputPaths) {
2539       builder.addCompactionOutput(outputPath.getName());
2540     }
2541     builder.setRegionName(ByteStringer.wrap(info.getRegionName()));
2542     return builder.build();
2543   }
2544 
2545   public static FlushDescriptor toFlushDescriptor(FlushAction action, HRegionInfo hri,
2546       long flushSeqId, Map<byte[], List<Path>> committedFiles) {
2547     FlushDescriptor.Builder desc = FlushDescriptor.newBuilder()
2548         .setAction(action)
2549         .setEncodedRegionName(ByteStringer.wrap(hri.getEncodedNameAsBytes()))
2550         .setFlushSequenceNumber(flushSeqId)
2551         .setTableName(ByteStringer.wrap(hri.getTable().getName()));
2552 
2553     for (Map.Entry<byte[], List<Path>> entry : committedFiles.entrySet()) {
2554       WALProtos.FlushDescriptor.StoreFlushDescriptor.Builder builder =
2555           WALProtos.FlushDescriptor.StoreFlushDescriptor.newBuilder()
2556           .setFamilyName(ByteStringer.wrap(entry.getKey()))
2557           .setStoreHomeDir(Bytes.toString(entry.getKey())); //relative to region
2558       if (entry.getValue() != null) {
2559         for (Path path : entry.getValue()) {
2560           builder.addFlushOutput(path.getName());
2561         }
2562       }
2563       desc.addStoreFlushes(builder);
2564     }
2565     return desc.build();
2566   }
2567 
2568   public static RegionEventDescriptor toRegionEventDescriptor(
2569       EventType eventType, HRegionInfo hri, long seqId, ServerName server,
2570       Map<byte[], List<Path>> storeFiles) {
2571     RegionEventDescriptor.Builder desc = RegionEventDescriptor.newBuilder()
2572         .setEventType(eventType)
2573         .setTableName(ByteStringer.wrap(hri.getTable().getName()))
2574         .setEncodedRegionName(ByteStringer.wrap(hri.getEncodedNameAsBytes()))
2575         .setLogSequenceNumber(seqId)
2576         .setServer(toServerName(server));
2577 
2578     for (Map.Entry<byte[], List<Path>> entry : storeFiles.entrySet()) {
2579       RegionEventDescriptor.StoreDescriptor.Builder builder
2580         = RegionEventDescriptor.StoreDescriptor.newBuilder()
2581           .setFamilyName(ByteStringer.wrap(entry.getKey()))
2582           .setStoreHomeDir(Bytes.toString(entry.getKey()));
2583       for (Path path : entry.getValue()) {
2584         builder.addStoreFile(path.getName());
2585       }
2586 
2587       desc.addStores(builder);
2588     }
2589     return desc.build();
2590   }
2591 
2592   /**
2593    * Return short version of Message toString'd, shorter than TextFormat#shortDebugString.
2594    * Tries to NOT print out data both because it can be big but also so we do not have data in our
2595    * logs. Use judiciously.
2596    * @param m
2597    * @return toString of passed <code>m</code>
2598    */
2599   public static String getShortTextFormat(Message m) {
2600     if (m == null) return "null";
2601     if (m instanceof ScanRequest) {
2602       // This should be small and safe to output.  No data.
2603       return TextFormat.shortDebugString(m);
2604     } else if (m instanceof RegionServerReportRequest) {
2605       // Print a short message only, just the servername and the requests, not the full load.
2606       RegionServerReportRequest r = (RegionServerReportRequest)m;
2607       return "server " + TextFormat.shortDebugString(r.getServer()) +
2608         " load { numberOfRequests: " + r.getLoad().getNumberOfRequests() + " }";
2609     } else if (m instanceof RegionServerStartupRequest) {
2610       // Should be small enough.
2611       return TextFormat.shortDebugString(m);
2612     } else if (m instanceof MutationProto) {
2613       return toShortString((MutationProto)m);
2614     } else if (m instanceof GetRequest) {
2615       GetRequest r = (GetRequest) m;
2616       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2617           ", row=" + getStringForByteString(r.getGet().getRow());
2618     } else if (m instanceof ClientProtos.MultiRequest) {
2619       ClientProtos.MultiRequest r = (ClientProtos.MultiRequest) m;
2620       // Get first set of Actions.
2621       ClientProtos.RegionAction actions = r.getRegionActionList().get(0);
2622       String row = actions.getActionCount() <= 0? "":
2623         getStringForByteString(actions.getAction(0).hasGet()?
2624           actions.getAction(0).getGet().getRow():
2625           actions.getAction(0).getMutation().getRow());
2626       return "region= " + getStringForByteString(actions.getRegion().getValue()) +
2627           ", for " + r.getRegionActionCount() +
2628           " actions and 1st row key=" + row;
2629     } else if (m instanceof ClientProtos.MutateRequest) {
2630       ClientProtos.MutateRequest r = (ClientProtos.MutateRequest) m;
2631       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2632           ", row=" + getStringForByteString(r.getMutation().getRow());
2633     }
2634     return "TODO: " + m.getClass().toString();
2635   }
2636 
2637   private static String getStringForByteString(ByteString bs) {
2638     return Bytes.toStringBinary(bs.toByteArray());
2639   }
2640 
2641   /**
2642    * Print out some subset of a MutationProto rather than all of it and its data
2643    * @param proto Protobuf to print out
2644    * @return Short String of mutation proto
2645    */
2646   static String toShortString(final MutationProto proto) {
2647     return "row=" + Bytes.toString(proto.getRow().toByteArray()) +
2648         ", type=" + proto.getMutateType().toString();
2649   }
2650 
2651   public static TableName toTableName(HBaseProtos.TableName tableNamePB) {
2652     return TableName.valueOf(tableNamePB.getNamespace().asReadOnlyByteBuffer(),
2653         tableNamePB.getQualifier().asReadOnlyByteBuffer());
2654   }
2655 
2656   public static HBaseProtos.TableName toProtoTableName(TableName tableName) {
2657     return HBaseProtos.TableName.newBuilder()
2658         .setNamespace(ByteStringer.wrap(tableName.getNamespace()))
2659         .setQualifier(ByteStringer.wrap(tableName.getQualifier())).build();
2660   }
2661 
2662   public static TableName[] getTableNameArray(List<HBaseProtos.TableName> tableNamesList) {
2663     if (tableNamesList == null) {
2664       return new TableName[0];
2665     }
2666     TableName[] tableNames = new TableName[tableNamesList.size()];
2667     for (int i = 0; i < tableNamesList.size(); i++) {
2668       tableNames[i] = toTableName(tableNamesList.get(i));
2669     }
2670     return tableNames;
2671   }
2672 
2673   /**
2674    * Convert a protocol buffer CellVisibility to a client CellVisibility
2675    *
2676    * @param proto
2677    * @return the converted client CellVisibility
2678    */
2679   public static CellVisibility toCellVisibility(ClientProtos.CellVisibility proto) {
2680     if (proto == null) return null;
2681     return new CellVisibility(proto.getExpression());
2682   }
2683 
2684   /**
2685    * Convert a protocol buffer CellVisibility bytes to a client CellVisibility
2686    *
2687    * @param protoBytes
2688    * @return the converted client CellVisibility
2689    * @throws DeserializationException
2690    */
2691   public static CellVisibility toCellVisibility(byte[] protoBytes) throws DeserializationException {
2692     if (protoBytes == null) return null;
2693     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2694     ClientProtos.CellVisibility proto = null;
2695     try {
2696       proto = builder.mergeFrom(protoBytes).build();
2697     } catch (InvalidProtocolBufferException e) {
2698       throw new DeserializationException(e);
2699     }
2700     return toCellVisibility(proto);
2701   }
2702 
2703   /**
2704    * Create a protocol buffer CellVisibility based on a client CellVisibility.
2705    *
2706    * @param cellVisibility
2707    * @return a protocol buffer CellVisibility
2708    */
2709   public static ClientProtos.CellVisibility toCellVisibility(CellVisibility cellVisibility) {
2710     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2711     builder.setExpression(cellVisibility.getExpression());
2712     return builder.build();
2713   }
2714 
2715   /**
2716    * Convert a protocol buffer Authorizations to a client Authorizations
2717    *
2718    * @param proto
2719    * @return the converted client Authorizations
2720    */
2721   public static Authorizations toAuthorizations(ClientProtos.Authorizations proto) {
2722     if (proto == null) return null;
2723     return new Authorizations(proto.getLabelList());
2724   }
2725 
2726   /**
2727    * Convert a protocol buffer Authorizations bytes to a client Authorizations
2728    *
2729    * @param protoBytes
2730    * @return the converted client Authorizations
2731    * @throws DeserializationException
2732    */
2733   public static Authorizations toAuthorizations(byte[] protoBytes) throws DeserializationException {
2734     if (protoBytes == null) return null;
2735     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2736     ClientProtos.Authorizations proto = null;
2737     try {
2738       proto = builder.mergeFrom(protoBytes).build();
2739     } catch (InvalidProtocolBufferException e) {
2740       throw new DeserializationException(e);
2741     }
2742     return toAuthorizations(proto);
2743   }
2744 
2745   /**
2746    * Create a protocol buffer Authorizations based on a client Authorizations.
2747    *
2748    * @param authorizations
2749    * @return a protocol buffer Authorizations
2750    */
2751   public static ClientProtos.Authorizations toAuthorizations(Authorizations authorizations) {
2752     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2753     for (String label : authorizations.getLabels()) {
2754       builder.addLabel(label);
2755     }
2756     return builder.build();
2757   }
2758 
2759   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(String user,
2760       Permission perms) {
2761     return AccessControlProtos.UsersAndPermissions.newBuilder()
2762       .addUserPermissions(AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder()
2763         .setUser(ByteString.copyFromUtf8(user))
2764         .addPermissions(toPermission(perms))
2765         .build())
2766       .build();
2767   }
2768 
2769   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(
2770       ListMultimap<String, Permission> perms) {
2771     AccessControlProtos.UsersAndPermissions.Builder builder =
2772         AccessControlProtos.UsersAndPermissions.newBuilder();
2773     for (Map.Entry<String, Collection<Permission>> entry : perms.asMap().entrySet()) {
2774       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
2775         AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
2776       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
2777       for (Permission perm: entry.getValue()) {
2778         userPermBuilder.addPermissions(toPermission(perm));
2779       }
2780       builder.addUserPermissions(userPermBuilder.build());
2781     }
2782     return builder.build();
2783   }
2784 
2785   public static ListMultimap<String, Permission> toUsersAndPermissions(
2786       AccessControlProtos.UsersAndPermissions proto) {
2787     ListMultimap<String, Permission> result = ArrayListMultimap.create();
2788     for (AccessControlProtos.UsersAndPermissions.UserPermissions userPerms:
2789         proto.getUserPermissionsList()) {
2790       String user = userPerms.getUser().toStringUtf8();
2791       for (AccessControlProtos.Permission perm: userPerms.getPermissionsList()) {
2792         result.put(user, toPermission(perm));
2793       }
2794     }
2795     return result;
2796   }
2797 }