View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distr=ibuted on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.protobuf;
19  
20  
21  import static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME;
22  
23  import java.io.ByteArrayOutputStream;
24  import java.io.IOException;
25  import java.lang.reflect.Constructor;
26  import java.lang.reflect.InvocationTargetException;
27  import java.lang.reflect.Method;
28  import java.lang.reflect.ParameterizedType;
29  import java.lang.reflect.Type;
30  import java.nio.ByteBuffer;
31  import java.util.ArrayList;
32  import java.util.Collection;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.Map.Entry;
37  import java.util.NavigableSet;
38  
39  import com.google.protobuf.HBaseZeroCopyByteString;
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.fs.Path;
42  import org.apache.hadoop.hbase.Cell;
43  import org.apache.hadoop.hbase.CellScanner;
44  import org.apache.hadoop.hbase.CellUtil;
45  import org.apache.hadoop.hbase.DoNotRetryIOException;
46  import org.apache.hadoop.hbase.HBaseConfiguration;
47  import org.apache.hadoop.hbase.HConstants;
48  import org.apache.hadoop.hbase.HRegionInfo;
49  import org.apache.hadoop.hbase.HTableDescriptor;
50  import org.apache.hadoop.hbase.KeyValue;
51  import org.apache.hadoop.hbase.KeyValueUtil;
52  import org.apache.hadoop.hbase.NamespaceDescriptor;
53  import org.apache.hadoop.hbase.ServerName;
54  import org.apache.hadoop.hbase.TableName;
55  import org.apache.hadoop.hbase.Tag;
56  import org.apache.hadoop.hbase.client.Append;
57  import org.apache.hadoop.hbase.client.Delete;
58  import org.apache.hadoop.hbase.client.Durability;
59  import org.apache.hadoop.hbase.client.Get;
60  import org.apache.hadoop.hbase.client.Increment;
61  import org.apache.hadoop.hbase.client.Mutation;
62  import org.apache.hadoop.hbase.client.Put;
63  import org.apache.hadoop.hbase.client.Result;
64  import org.apache.hadoop.hbase.client.Scan;
65  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
66  import org.apache.hadoop.hbase.exceptions.DeserializationException;
67  import org.apache.hadoop.hbase.filter.ByteArrayComparable;
68  import org.apache.hadoop.hbase.filter.Filter;
69  import org.apache.hadoop.hbase.io.TimeRange;
70  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
71  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService;
72  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
73  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
74  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
75  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
76  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
77  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
78  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
79  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
80  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
81  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
82  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
83  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
84  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
85  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
86  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
87  import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
88  import org.apache.hadoop.hbase.protobuf.generated.CellProtos;
89  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
90  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
91  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
92  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
93  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Column;
94  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall;
95  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
96  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
97  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
98  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
99  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
100 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue;
101 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.ColumnValue.QualifierValue;
102 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.DeleteType;
103 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
104 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
105 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
106 import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
107 import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
108 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
109 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
110 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo;
111 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
112 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
113 import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
114 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
115 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
116 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService;
117 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
118 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
119 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
120 import org.apache.hadoop.hbase.security.access.Permission;
121 import org.apache.hadoop.hbase.security.access.TablePermission;
122 import org.apache.hadoop.hbase.security.access.UserPermission;
123 import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier;
124 import org.apache.hadoop.hbase.security.visibility.Authorizations;
125 import org.apache.hadoop.hbase.security.visibility.CellVisibility;
126 import org.apache.hadoop.hbase.util.Bytes;
127 import org.apache.hadoop.hbase.util.DynamicClassLoader;
128 import org.apache.hadoop.hbase.util.ExceptionUtil;
129 import org.apache.hadoop.hbase.util.Methods;
130 import org.apache.hadoop.hbase.util.Pair;
131 import org.apache.hadoop.io.Text;
132 import org.apache.hadoop.ipc.RemoteException;
133 import org.apache.hadoop.security.token.Token;
134 
135 import com.google.common.collect.ArrayListMultimap;
136 import com.google.common.collect.ListMultimap;
137 import com.google.common.collect.Lists;
138 import com.google.protobuf.ByteString;
139 import com.google.protobuf.InvalidProtocolBufferException;
140 import com.google.protobuf.Message;
141 import com.google.protobuf.Parser;
142 import com.google.protobuf.RpcChannel;
143 import com.google.protobuf.Service;
144 import com.google.protobuf.ServiceException;
145 import com.google.protobuf.TextFormat;
146 
147 /**
148  * Protobufs utility.
149  */
150 public final class ProtobufUtil {
151 
152   private ProtobufUtil() {
153   }
154 
155   /**
156    * Primitive type to class mapping.
157    */
158   private final static Map<String, Class<?>>
159     PRIMITIVES = new HashMap<String, Class<?>>();
160 
161 
162   /**
163    * Many results are simple: no cell, exists true or false. To save on object creations,
164    *  we reuse them across calls.
165    */
166   private final static Cell[] EMPTY_CELL_ARRAY = new Cell[]{};
167   private final static Result EMPTY_RESULT = Result.create(EMPTY_CELL_ARRAY);
168   private final static Result EMPTY_RESULT_EXISTS_TRUE = Result.create(null, true);
169   private final static Result EMPTY_RESULT_EXISTS_FALSE = Result.create(null, false);
170 
171   private final static ClientProtos.Result EMPTY_RESULT_PB;
172   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_TRUE;
173   private final static ClientProtos.Result EMPTY_RESULT_PB_EXISTS_FALSE;
174 
175   static {
176     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
177 
178     builder.setExists(true);
179     builder.setAssociatedCellCount(0);
180     EMPTY_RESULT_PB_EXISTS_TRUE =  builder.build();
181 
182     builder.clear();
183 
184     builder.setExists(false);
185     builder.setAssociatedCellCount(0);
186     EMPTY_RESULT_PB_EXISTS_FALSE =  builder.build();
187 
188     builder.clear();
189     builder.setAssociatedCellCount(0);
190     EMPTY_RESULT_PB =  builder.build();
191   }
192 
193   /**
194    * Dynamic class loader to load filter/comparators
195    */
196   private final static ClassLoader CLASS_LOADER;
197 
198   static {
199     ClassLoader parent = ProtobufUtil.class.getClassLoader();
200     Configuration conf = HBaseConfiguration.create();
201     CLASS_LOADER = new DynamicClassLoader(conf, parent);
202 
203     PRIMITIVES.put(Boolean.TYPE.getName(), Boolean.TYPE);
204     PRIMITIVES.put(Byte.TYPE.getName(), Byte.TYPE);
205     PRIMITIVES.put(Character.TYPE.getName(), Character.TYPE);
206     PRIMITIVES.put(Short.TYPE.getName(), Short.TYPE);
207     PRIMITIVES.put(Integer.TYPE.getName(), Integer.TYPE);
208     PRIMITIVES.put(Long.TYPE.getName(), Long.TYPE);
209     PRIMITIVES.put(Float.TYPE.getName(), Float.TYPE);
210     PRIMITIVES.put(Double.TYPE.getName(), Double.TYPE);
211     PRIMITIVES.put(Void.TYPE.getName(), Void.TYPE);
212   }
213 
214   /**
215    * Magic we put ahead of a serialized protobuf message.
216    * For example, all znode content is protobuf messages with the below magic
217    * for preamble.
218    */
219   public static final byte [] PB_MAGIC = new byte [] {'P', 'B', 'U', 'F'};
220   private static final String PB_MAGIC_STR = Bytes.toString(PB_MAGIC);
221 
222   /**
223    * Prepend the passed bytes with four bytes of magic, {@link #PB_MAGIC}, to flag what
224    * follows as a protobuf in hbase.  Prepend these bytes to all content written to znodes, etc.
225    * @param bytes Bytes to decorate
226    * @return The passed <code>bytes</codes> with magic prepended (Creates a new
227    * byte array that is <code>bytes.length</code> plus {@link #PB_MAGIC}.length.
228    */
229   public static byte [] prependPBMagic(final byte [] bytes) {
230     return Bytes.add(PB_MAGIC, bytes);
231   }
232 
233   /**
234    * @param bytes Bytes to check.
235    * @return True if passed <code>bytes</code> has {@link #PB_MAGIC} for a prefix.
236    */
237   public static boolean isPBMagicPrefix(final byte [] bytes) {
238     return isPBMagicPrefix(bytes, 0, bytes.length);
239   }
240 
241   /**
242    * @param bytes Bytes to check.
243    * @return True if passed <code>bytes</code> has {@link #PB_MAGIC} for a prefix.
244    */
245   public static boolean isPBMagicPrefix(final byte [] bytes, int offset, int len) {
246     if (bytes == null || len < PB_MAGIC.length) return false;
247     return Bytes.compareTo(PB_MAGIC, 0, PB_MAGIC.length, bytes, offset, PB_MAGIC.length) == 0;
248   }
249 
250   /**
251    * @param bytes
252    * @throws DeserializationException if we are missing the pb magic prefix
253    */
254   public static void expectPBMagicPrefix(final byte [] bytes) throws DeserializationException {
255     if (!isPBMagicPrefix(bytes)) {
256       throw new DeserializationException("Missing pb magic " + PB_MAGIC_STR + " prefix");
257     }
258   }
259 
260   /**
261    * @return Length of {@link #PB_MAGIC}
262    */
263   public static int lengthOfPBMagic() {
264     return PB_MAGIC.length;
265   }
266 
267   /**
268    * Return the IOException thrown by the remote server wrapped in
269    * ServiceException as cause.
270    *
271    * @param se ServiceException that wraps IO exception thrown by the server
272    * @return Exception wrapped in ServiceException or
273    *   a new IOException that wraps the unexpected ServiceException.
274    */
275   public static IOException getRemoteException(ServiceException se) {
276     Throwable e = se.getCause();
277     if (e == null) {
278       return new IOException(se);
279     }
280     if (ExceptionUtil.isInterrupt(e)) {
281       return ExceptionUtil.asInterrupt(e);
282     }
283     if (e instanceof RemoteException) {
284       e = ((RemoteException) e).unwrapRemoteException();
285     }
286     return e instanceof IOException ? (IOException) e : new IOException(se);
287   }
288 
289   /**
290    * Convert a ServerName to a protocol buffer ServerName
291    *
292    * @param serverName the ServerName to convert
293    * @return the converted protocol buffer ServerName
294    * @see #toServerName(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName)
295    */
296   public static HBaseProtos.ServerName
297       toServerName(final ServerName serverName) {
298     if (serverName == null) return null;
299     HBaseProtos.ServerName.Builder builder =
300       HBaseProtos.ServerName.newBuilder();
301     builder.setHostName(serverName.getHostname());
302     if (serverName.getPort() >= 0) {
303       builder.setPort(serverName.getPort());
304     }
305     if (serverName.getStartcode() >= 0) {
306       builder.setStartCode(serverName.getStartcode());
307     }
308     return builder.build();
309   }
310 
311   /**
312    * Convert a protocol buffer ServerName to a ServerName
313    *
314    * @param proto the protocol buffer ServerName to convert
315    * @return the converted ServerName
316    */
317   public static ServerName toServerName(final HBaseProtos.ServerName proto) {
318     if (proto == null) return null;
319     String hostName = proto.getHostName();
320     long startCode = -1;
321     int port = -1;
322     if (proto.hasPort()) {
323       port = proto.getPort();
324     }
325     if (proto.hasStartCode()) {
326       startCode = proto.getStartCode();
327     }
328     return ServerName.valueOf(hostName, port, startCode);
329   }
330 
331   /**
332    * Get HTableDescriptor[] from GetTableDescriptorsResponse protobuf
333    *
334    * @param proto the GetTableDescriptorsResponse
335    * @return HTableDescriptor[]
336    */
337   public static HTableDescriptor[] getHTableDescriptorArray(GetTableDescriptorsResponse proto) {
338     if (proto == null) return null;
339 
340     HTableDescriptor[] ret = new HTableDescriptor[proto.getTableSchemaCount()];
341     for (int i = 0; i < proto.getTableSchemaCount(); ++i) {
342       ret[i] = HTableDescriptor.convert(proto.getTableSchema(i));
343     }
344     return ret;
345   }
346 
347   /**
348    * get the split keys in form "byte [][]" from a CreateTableRequest proto
349    *
350    * @param proto the CreateTableRequest
351    * @return the split keys
352    */
353   public static byte [][] getSplitKeysArray(final CreateTableRequest proto) {
354     byte [][] splitKeys = new byte[proto.getSplitKeysCount()][];
355     for (int i = 0; i < proto.getSplitKeysCount(); ++i) {
356       splitKeys[i] = proto.getSplitKeys(i).toByteArray();
357     }
358     return splitKeys;
359   }
360 
361   /**
362    * Convert a protobuf Durability into a client Durability
363    */
364   public static Durability toDurability(
365       final ClientProtos.MutationProto.Durability proto) {
366     switch(proto) {
367     case USE_DEFAULT:
368       return Durability.USE_DEFAULT;
369     case SKIP_WAL:
370       return Durability.SKIP_WAL;
371     case ASYNC_WAL:
372       return Durability.ASYNC_WAL;
373     case SYNC_WAL:
374       return Durability.SYNC_WAL;
375     case FSYNC_WAL:
376       return Durability.FSYNC_WAL;
377     default:
378       return Durability.USE_DEFAULT;
379     }
380   }
381 
382   /**
383    * Convert a client Durability into a protbuf Durability
384    */
385   public static ClientProtos.MutationProto.Durability toDurability(
386       final Durability d) {
387     switch(d) {
388     case USE_DEFAULT:
389       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
390     case SKIP_WAL:
391       return ClientProtos.MutationProto.Durability.SKIP_WAL;
392     case ASYNC_WAL:
393       return ClientProtos.MutationProto.Durability.ASYNC_WAL;
394     case SYNC_WAL:
395       return ClientProtos.MutationProto.Durability.SYNC_WAL;
396     case FSYNC_WAL:
397       return ClientProtos.MutationProto.Durability.FSYNC_WAL;
398     default:
399       return ClientProtos.MutationProto.Durability.USE_DEFAULT;
400     }
401   }
402 
403   /**
404    * Convert a protocol buffer Get to a client Get
405    *
406    * @param proto the protocol buffer Get to convert
407    * @return the converted client Get
408    * @throws IOException
409    */
410   public static Get toGet(
411       final ClientProtos.Get proto) throws IOException {
412     if (proto == null) return null;
413     byte[] row = proto.getRow().toByteArray();
414     Get get = new Get(row);
415     if (proto.hasCacheBlocks()) {
416       get.setCacheBlocks(proto.getCacheBlocks());
417     }
418     if (proto.hasMaxVersions()) {
419       get.setMaxVersions(proto.getMaxVersions());
420     }
421     if (proto.hasStoreLimit()) {
422       get.setMaxResultsPerColumnFamily(proto.getStoreLimit());
423     }
424     if (proto.hasStoreOffset()) {
425       get.setRowOffsetPerColumnFamily(proto.getStoreOffset());
426     }
427     if (proto.hasTimeRange()) {
428       HBaseProtos.TimeRange timeRange = proto.getTimeRange();
429       long minStamp = 0;
430       long maxStamp = Long.MAX_VALUE;
431       if (timeRange.hasFrom()) {
432         minStamp = timeRange.getFrom();
433       }
434       if (timeRange.hasTo()) {
435         maxStamp = timeRange.getTo();
436       }
437       get.setTimeRange(minStamp, maxStamp);
438     }
439     if (proto.hasFilter()) {
440       FilterProtos.Filter filter = proto.getFilter();
441       get.setFilter(ProtobufUtil.toFilter(filter));
442     }
443     for (NameBytesPair attribute: proto.getAttributeList()) {
444       get.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
445     }
446     if (proto.getColumnCount() > 0) {
447       for (Column column: proto.getColumnList()) {
448         byte[] family = column.getFamily().toByteArray();
449         if (column.getQualifierCount() > 0) {
450           for (ByteString qualifier: column.getQualifierList()) {
451             get.addColumn(family, qualifier.toByteArray());
452           }
453         } else {
454           get.addFamily(family);
455         }
456       }
457     }
458     if (proto.hasExistenceOnly() && proto.getExistenceOnly()){
459       get.setCheckExistenceOnly(true);
460     }
461     if (proto.hasClosestRowBefore() && proto.getClosestRowBefore()){
462       get.setClosestRowBefore(true);
463     }
464     return get;
465   }
466 
467   /**
468    * Convert a protocol buffer Mutate to a Put.
469    *
470    * @param proto The protocol buffer MutationProto to convert
471    * @return A client Put.
472    * @throws IOException
473    */
474   public static Put toPut(final MutationProto proto)
475   throws IOException {
476     return toPut(proto, null);
477   }
478 
479   /**
480    * Convert a protocol buffer Mutate to a Put.
481    *
482    * @param proto The protocol buffer MutationProto to convert
483    * @param cellScanner If non-null, the Cell data that goes with this proto.
484    * @return A client Put.
485    * @throws IOException
486    */
487   public static Put toPut(final MutationProto proto, final CellScanner cellScanner)
488   throws IOException {
489     // TODO: Server-side at least why do we convert back to the Client types?  Why not just pb it?
490     MutationType type = proto.getMutateType();
491     assert type == MutationType.PUT: type.name();
492     long timestamp = proto.hasTimestamp()? proto.getTimestamp(): HConstants.LATEST_TIMESTAMP;
493     Put put = null;
494     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
495     if (cellCount > 0) {
496       // The proto has metadata only and the data is separate to be found in the cellScanner.
497       if (cellScanner == null) {
498         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
499             toShortString(proto));
500       }
501       for (int i = 0; i < cellCount; i++) {
502         if (!cellScanner.advance()) {
503           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
504             " no cell returned: " + toShortString(proto));
505         }
506         Cell cell = cellScanner.current();
507         if (put == null) {
508           put = new Put(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
509         }
510         put.add(cell);
511       }
512     } else {
513       if (proto.hasRow()) {
514         put = new Put(proto.getRow().asReadOnlyByteBuffer(), timestamp);
515       } else {
516         throw new IllegalArgumentException("row cannot be null");
517       }
518       // The proto has the metadata and the data itself
519       for (ColumnValue column: proto.getColumnValueList()) {
520         byte[] family = column.getFamily().toByteArray();
521         for (QualifierValue qv: column.getQualifierValueList()) {
522           if (!qv.hasValue()) {
523             throw new DoNotRetryIOException(
524                 "Missing required field: qualifier value");
525           }
526           ByteBuffer qualifier =
527               qv.hasQualifier() ? qv.getQualifier().asReadOnlyByteBuffer() : null;
528           ByteBuffer value =
529               qv.hasValue() ? qv.getValue().asReadOnlyByteBuffer() : null;
530           long ts = timestamp;
531           if (qv.hasTimestamp()) {
532             ts = qv.getTimestamp();
533           }
534           byte[] tags;
535           if (qv.hasTags()) {
536             tags = qv.getTags().toByteArray();
537             Object[] array = Tag.asList(tags, 0, (short)tags.length).toArray();
538             Tag[] tagArray = new Tag[array.length];
539             for(int i = 0; i< array.length; i++) {
540               tagArray[i] = (Tag)array[i];
541             }
542             put.addImmutable(family, qualifier, ts, value, tagArray);
543           } else {
544             put.addImmutable(family, qualifier, ts, value);
545           }
546         }
547       }
548     }
549     put.setDurability(toDurability(proto.getDurability()));
550     for (NameBytesPair attribute: proto.getAttributeList()) {
551       put.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
552     }
553     return put;
554   }
555 
556   /**
557    * Convert a protocol buffer Mutate to a Delete
558    *
559    * @param proto the protocol buffer Mutate to convert
560    * @return the converted client Delete
561    * @throws IOException
562    */
563   public static Delete toDelete(final MutationProto proto)
564   throws IOException {
565     return toDelete(proto, null);
566   }
567 
568   /**
569    * Convert a protocol buffer Mutate to a Delete
570    *
571    * @param proto the protocol buffer Mutate to convert
572    * @param cellScanner if non-null, the data that goes with this delete.
573    * @return the converted client Delete
574    * @throws IOException
575    */
576   public static Delete toDelete(final MutationProto proto, final CellScanner cellScanner)
577   throws IOException {
578     MutationType type = proto.getMutateType();
579     assert type == MutationType.DELETE : type.name();
580     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
581     long timestamp = HConstants.LATEST_TIMESTAMP;
582     if (proto.hasTimestamp()) {
583       timestamp = proto.getTimestamp();
584     }
585     Delete delete = null;
586     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
587     if (cellCount > 0) {
588       // The proto has metadata only and the data is separate to be found in the cellScanner.
589       if (cellScanner == null) {
590         // TextFormat should be fine for a Delete since it carries no data, just coordinates.
591         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
592           TextFormat.shortDebugString(proto));
593       }
594       for (int i = 0; i < cellCount; i++) {
595         if (!cellScanner.advance()) {
596           // TextFormat should be fine for a Delete since it carries no data, just coordinates.
597           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
598             " no cell returned: " + TextFormat.shortDebugString(proto));
599         }
600         Cell cell = cellScanner.current();
601         if (delete == null) {
602           delete =
603             new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
604         }
605         delete.addDeleteMarker(KeyValueUtil.ensureKeyValue(cell));
606       }
607     } else {
608       delete = new Delete(row, timestamp);
609       for (ColumnValue column: proto.getColumnValueList()) {
610         byte[] family = column.getFamily().toByteArray();
611         for (QualifierValue qv: column.getQualifierValueList()) {
612           DeleteType deleteType = qv.getDeleteType();
613           byte[] qualifier = null;
614           if (qv.hasQualifier()) {
615             qualifier = qv.getQualifier().toByteArray();
616           }
617           long ts = HConstants.LATEST_TIMESTAMP;
618           if (qv.hasTimestamp()) {
619             ts = qv.getTimestamp();
620           }
621           if (deleteType == DeleteType.DELETE_ONE_VERSION) {
622             delete.deleteColumn(family, qualifier, ts);
623           } else if (deleteType == DeleteType.DELETE_MULTIPLE_VERSIONS) {
624             delete.deleteColumns(family, qualifier, ts);
625           } else if (deleteType == DeleteType.DELETE_FAMILY_VERSION) {
626             delete.deleteFamilyVersion(family, ts);
627           } else {
628             delete.deleteFamily(family, ts);
629           }
630         }
631       }
632     }
633     delete.setDurability(toDurability(proto.getDurability()));
634     for (NameBytesPair attribute: proto.getAttributeList()) {
635       delete.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
636     }
637     return delete;
638   }
639 
640   /**
641    * Convert a protocol buffer Mutate to an Append
642    * @param cellScanner
643    * @param proto the protocol buffer Mutate to convert
644    * @return the converted client Append
645    * @throws IOException
646    */
647   public static Append toAppend(final MutationProto proto, final CellScanner cellScanner)
648   throws IOException {
649     MutationType type = proto.getMutateType();
650     assert type == MutationType.APPEND : type.name();
651     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
652     Append append = null;
653     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
654     if (cellCount > 0) {
655       // The proto has metadata only and the data is separate to be found in the cellScanner.
656       if (cellScanner == null) {
657         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
658           toShortString(proto));
659       }
660       for (int i = 0; i < cellCount; i++) {
661         if (!cellScanner.advance()) {
662           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
663             " no cell returned: " + toShortString(proto));
664         }
665         Cell cell = cellScanner.current();
666         if (append == null) {
667           append = new Append(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
668         }
669         append.add(KeyValueUtil.ensureKeyValue(cell));
670       }
671     } else {
672       append = new Append(row);
673       for (ColumnValue column: proto.getColumnValueList()) {
674         byte[] family = column.getFamily().toByteArray();
675         for (QualifierValue qv: column.getQualifierValueList()) {
676           byte[] qualifier = qv.getQualifier().toByteArray();
677           if (!qv.hasValue()) {
678             throw new DoNotRetryIOException(
679               "Missing required field: qualifier value");
680           }
681           byte[] value = qv.getValue().toByteArray();
682           byte[] tags = null;
683           if (qv.hasTags()) {
684             tags = qv.getTags().toByteArray();
685           }
686           append.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
687               KeyValue.Type.Put, value, tags));
688         }
689       }
690     }
691     append.setDurability(toDurability(proto.getDurability()));
692     for (NameBytesPair attribute: proto.getAttributeList()) {
693       append.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
694     }
695     return append;
696   }
697 
698   /**
699    * Convert a MutateRequest to Mutation
700    *
701    * @param proto the protocol buffer Mutate to convert
702    * @return the converted Mutation
703    * @throws IOException
704    */
705   public static Mutation toMutation(final MutationProto proto) throws IOException {
706     MutationType type = proto.getMutateType();
707     if (type == MutationType.APPEND) {
708       return toAppend(proto, null);
709     }
710     if (type == MutationType.DELETE) {
711       return toDelete(proto, null);
712     }
713     if (type == MutationType.PUT) {
714       return toPut(proto, null);
715     }
716     throw new IOException("Unknown mutation type " + type);
717   }
718 
719   /**
720    * Convert a protocol buffer Mutate to an Increment
721    *
722    * @param proto the protocol buffer Mutate to convert
723    * @return the converted client Increment
724    * @throws IOException
725    */
726   public static Increment toIncrement(final MutationProto proto, final CellScanner cellScanner)
727   throws IOException {
728     MutationType type = proto.getMutateType();
729     assert type == MutationType.INCREMENT : type.name();
730     byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
731     Increment increment = null;
732     int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
733     if (cellCount > 0) {
734       // The proto has metadata only and the data is separate to be found in the cellScanner.
735       if (cellScanner == null) {
736         throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " +
737           TextFormat.shortDebugString(proto));
738       }
739       for (int i = 0; i < cellCount; i++) {
740         if (!cellScanner.advance()) {
741           throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i +
742             " no cell returned: " + TextFormat.shortDebugString(proto));
743         }
744         Cell cell = cellScanner.current();
745         if (increment == null) {
746           increment = new Increment(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
747         }
748         increment.add(KeyValueUtil.ensureKeyValue(cell));
749       }
750     } else {
751       increment = new Increment(row);
752       for (ColumnValue column: proto.getColumnValueList()) {
753         byte[] family = column.getFamily().toByteArray();
754         for (QualifierValue qv: column.getQualifierValueList()) {
755           byte[] qualifier = qv.getQualifier().toByteArray();
756           if (!qv.hasValue()) {
757             throw new DoNotRetryIOException("Missing required field: qualifier value");
758           }
759           byte[] value = qv.getValue().toByteArray();
760           byte[] tags = null;
761           if (qv.hasTags()) {
762             tags = qv.getTags().toByteArray();
763           }
764           increment.add(CellUtil.createCell(row, family, qualifier, qv.getTimestamp(),
765               KeyValue.Type.Put, value, tags));
766         }
767       }
768     }
769     if (proto.hasTimeRange()) {
770       HBaseProtos.TimeRange timeRange = proto.getTimeRange();
771       long minStamp = 0;
772       long maxStamp = Long.MAX_VALUE;
773       if (timeRange.hasFrom()) {
774         minStamp = timeRange.getFrom();
775       }
776       if (timeRange.hasTo()) {
777         maxStamp = timeRange.getTo();
778       }
779       increment.setTimeRange(minStamp, maxStamp);
780     }
781     increment.setDurability(toDurability(proto.getDurability()));
782     for (NameBytesPair attribute : proto.getAttributeList()) {
783       increment.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
784     }
785     return increment;
786   }
787 
788   /**
789    * Convert a client Scan to a protocol buffer Scan
790    *
791    * @param scan the client Scan to convert
792    * @return the converted protocol buffer Scan
793    * @throws IOException
794    */
795   public static ClientProtos.Scan toScan(
796       final Scan scan) throws IOException {
797     ClientProtos.Scan.Builder scanBuilder =
798       ClientProtos.Scan.newBuilder();
799     scanBuilder.setCacheBlocks(scan.getCacheBlocks());
800     if (scan.getBatch() > 0) {
801       scanBuilder.setBatchSize(scan.getBatch());
802     }
803     if (scan.getMaxResultSize() > 0) {
804       scanBuilder.setMaxResultSize(scan.getMaxResultSize());
805     }
806     if (scan.isSmall()) {
807       scanBuilder.setSmall(scan.isSmall());
808     }
809     Boolean loadColumnFamiliesOnDemand = scan.getLoadColumnFamiliesOnDemandValue();
810     if (loadColumnFamiliesOnDemand != null) {
811       scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand.booleanValue());
812     }
813     scanBuilder.setMaxVersions(scan.getMaxVersions());
814     TimeRange timeRange = scan.getTimeRange();
815     if (!timeRange.isAllTime()) {
816       HBaseProtos.TimeRange.Builder timeRangeBuilder =
817         HBaseProtos.TimeRange.newBuilder();
818       timeRangeBuilder.setFrom(timeRange.getMin());
819       timeRangeBuilder.setTo(timeRange.getMax());
820       scanBuilder.setTimeRange(timeRangeBuilder.build());
821     }
822     Map<String, byte[]> attributes = scan.getAttributesMap();
823     if (!attributes.isEmpty()) {
824       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
825       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
826         attributeBuilder.setName(attribute.getKey());
827         attributeBuilder.setValue(HBaseZeroCopyByteString.wrap(attribute.getValue()));
828         scanBuilder.addAttribute(attributeBuilder.build());
829       }
830     }
831     byte[] startRow = scan.getStartRow();
832     if (startRow != null && startRow.length > 0) {
833       scanBuilder.setStartRow(HBaseZeroCopyByteString.wrap(startRow));
834     }
835     byte[] stopRow = scan.getStopRow();
836     if (stopRow != null && stopRow.length > 0) {
837       scanBuilder.setStopRow(HBaseZeroCopyByteString.wrap(stopRow));
838     }
839     if (scan.hasFilter()) {
840       scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
841     }
842     if (scan.hasFamilies()) {
843       Column.Builder columnBuilder = Column.newBuilder();
844       for (Map.Entry<byte[],NavigableSet<byte []>>
845           family: scan.getFamilyMap().entrySet()) {
846         columnBuilder.setFamily(HBaseZeroCopyByteString.wrap(family.getKey()));
847         NavigableSet<byte []> qualifiers = family.getValue();
848         columnBuilder.clearQualifier();
849         if (qualifiers != null && qualifiers.size() > 0) {
850           for (byte [] qualifier: qualifiers) {
851             columnBuilder.addQualifier(HBaseZeroCopyByteString.wrap(qualifier));
852           }
853         }
854         scanBuilder.addColumn(columnBuilder.build());
855       }
856     }
857     if (scan.getMaxResultsPerColumnFamily() >= 0) {
858       scanBuilder.setStoreLimit(scan.getMaxResultsPerColumnFamily());
859     }
860     if (scan.getRowOffsetPerColumnFamily() > 0) {
861       scanBuilder.setStoreOffset(scan.getRowOffsetPerColumnFamily());
862     }
863     if (scan.isReversed()) {
864       scanBuilder.setReversed(scan.isReversed());
865     }
866     return scanBuilder.build();
867   }
868 
869   /**
870    * Convert a protocol buffer Scan to a client Scan
871    *
872    * @param proto the protocol buffer Scan to convert
873    * @return the converted client Scan
874    * @throws IOException
875    */
876   public static Scan toScan(
877       final ClientProtos.Scan proto) throws IOException {
878     byte [] startRow = HConstants.EMPTY_START_ROW;
879     byte [] stopRow  = HConstants.EMPTY_END_ROW;
880     if (proto.hasStartRow()) {
881       startRow = proto.getStartRow().toByteArray();
882     }
883     if (proto.hasStopRow()) {
884       stopRow = proto.getStopRow().toByteArray();
885     }
886     Scan scan = new Scan(startRow, stopRow);
887     if (proto.hasCacheBlocks()) {
888       scan.setCacheBlocks(proto.getCacheBlocks());
889     }
890     if (proto.hasMaxVersions()) {
891       scan.setMaxVersions(proto.getMaxVersions());
892     }
893     if (proto.hasStoreLimit()) {
894       scan.setMaxResultsPerColumnFamily(proto.getStoreLimit());
895     }
896     if (proto.hasStoreOffset()) {
897       scan.setRowOffsetPerColumnFamily(proto.getStoreOffset());
898     }
899     if (proto.hasLoadColumnFamiliesOnDemand()) {
900       scan.setLoadColumnFamiliesOnDemand(proto.getLoadColumnFamiliesOnDemand());
901     }
902     if (proto.hasTimeRange()) {
903       HBaseProtos.TimeRange timeRange = proto.getTimeRange();
904       long minStamp = 0;
905       long maxStamp = Long.MAX_VALUE;
906       if (timeRange.hasFrom()) {
907         minStamp = timeRange.getFrom();
908       }
909       if (timeRange.hasTo()) {
910         maxStamp = timeRange.getTo();
911       }
912       scan.setTimeRange(minStamp, maxStamp);
913     }
914     if (proto.hasFilter()) {
915       FilterProtos.Filter filter = proto.getFilter();
916       scan.setFilter(ProtobufUtil.toFilter(filter));
917     }
918     if (proto.hasBatchSize()) {
919       scan.setBatch(proto.getBatchSize());
920     }
921     if (proto.hasMaxResultSize()) {
922       scan.setMaxResultSize(proto.getMaxResultSize());
923     }
924     if (proto.hasSmall()) {
925       scan.setSmall(proto.getSmall());
926     }
927     for (NameBytesPair attribute: proto.getAttributeList()) {
928       scan.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
929     }
930     if (proto.getColumnCount() > 0) {
931       for (Column column: proto.getColumnList()) {
932         byte[] family = column.getFamily().toByteArray();
933         if (column.getQualifierCount() > 0) {
934           for (ByteString qualifier: column.getQualifierList()) {
935             scan.addColumn(family, qualifier.toByteArray());
936           }
937         } else {
938           scan.addFamily(family);
939         }
940       }
941     }
942     if (proto.hasReversed()) {
943       scan.setReversed(proto.getReversed());
944     }
945     return scan;
946   }
947 
948   /**
949    * Create a protocol buffer Get based on a client Get.
950    *
951    * @param get the client Get
952    * @return a protocol buffer Get
953    * @throws IOException
954    */
955   public static ClientProtos.Get toGet(
956       final Get get) throws IOException {
957     ClientProtos.Get.Builder builder =
958       ClientProtos.Get.newBuilder();
959     builder.setRow(HBaseZeroCopyByteString.wrap(get.getRow()));
960     builder.setCacheBlocks(get.getCacheBlocks());
961     builder.setMaxVersions(get.getMaxVersions());
962     if (get.getFilter() != null) {
963       builder.setFilter(ProtobufUtil.toFilter(get.getFilter()));
964     }
965     TimeRange timeRange = get.getTimeRange();
966     if (!timeRange.isAllTime()) {
967       HBaseProtos.TimeRange.Builder timeRangeBuilder =
968         HBaseProtos.TimeRange.newBuilder();
969       timeRangeBuilder.setFrom(timeRange.getMin());
970       timeRangeBuilder.setTo(timeRange.getMax());
971       builder.setTimeRange(timeRangeBuilder.build());
972     }
973     Map<String, byte[]> attributes = get.getAttributesMap();
974     if (!attributes.isEmpty()) {
975       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
976       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
977         attributeBuilder.setName(attribute.getKey());
978         attributeBuilder.setValue(HBaseZeroCopyByteString.wrap(attribute.getValue()));
979         builder.addAttribute(attributeBuilder.build());
980       }
981     }
982     if (get.hasFamilies()) {
983       Column.Builder columnBuilder = Column.newBuilder();
984       Map<byte[], NavigableSet<byte[]>> families = get.getFamilyMap();
985       for (Map.Entry<byte[], NavigableSet<byte[]>> family: families.entrySet()) {
986         NavigableSet<byte[]> qualifiers = family.getValue();
987         columnBuilder.setFamily(HBaseZeroCopyByteString.wrap(family.getKey()));
988         columnBuilder.clearQualifier();
989         if (qualifiers != null && qualifiers.size() > 0) {
990           for (byte[] qualifier: qualifiers) {
991             columnBuilder.addQualifier(HBaseZeroCopyByteString.wrap(qualifier));
992           }
993         }
994         builder.addColumn(columnBuilder.build());
995       }
996     }
997     if (get.getMaxResultsPerColumnFamily() >= 0) {
998       builder.setStoreLimit(get.getMaxResultsPerColumnFamily());
999     }
1000     if (get.getRowOffsetPerColumnFamily() > 0) {
1001       builder.setStoreOffset(get.getRowOffsetPerColumnFamily());
1002     }
1003     if (get.isCheckExistenceOnly()){
1004       builder.setExistenceOnly(true);
1005     }
1006     if (get.isClosestRowBefore()){
1007       builder.setClosestRowBefore(true);
1008     }
1009     return builder.build();
1010   }
1011 
1012   /**
1013    * Convert a client Increment to a protobuf Mutate.
1014    *
1015    * @param increment
1016    * @return the converted mutate
1017    */
1018   public static MutationProto toMutation(
1019     final Increment increment, final MutationProto.Builder builder, long nonce) {
1020     builder.setRow(HBaseZeroCopyByteString.wrap(increment.getRow()));
1021     builder.setMutateType(MutationType.INCREMENT);
1022     builder.setDurability(toDurability(increment.getDurability()));
1023     if (nonce != HConstants.NO_NONCE) {
1024       builder.setNonce(nonce);
1025     }
1026     TimeRange timeRange = increment.getTimeRange();
1027     if (!timeRange.isAllTime()) {
1028       HBaseProtos.TimeRange.Builder timeRangeBuilder =
1029         HBaseProtos.TimeRange.newBuilder();
1030       timeRangeBuilder.setFrom(timeRange.getMin());
1031       timeRangeBuilder.setTo(timeRange.getMax());
1032       builder.setTimeRange(timeRangeBuilder.build());
1033     }
1034     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1035     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1036     for (Map.Entry<byte[], List<Cell>> family: increment.getFamilyCellMap().entrySet()) {
1037       columnBuilder.setFamily(HBaseZeroCopyByteString.wrap(family.getKey()));
1038       columnBuilder.clearQualifierValue();
1039       List<Cell> values = family.getValue();
1040       if (values != null && values.size() > 0) {
1041         for (Cell cell: values) {
1042           KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
1043           valueBuilder.setQualifier(HBaseZeroCopyByteString.wrap(
1044               kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()));
1045           valueBuilder.setValue(HBaseZeroCopyByteString.wrap(
1046               kv.getValueArray(), kv.getValueOffset(), kv.getValueLength()));
1047           if (kv.getTagsLength() > 0) {
1048             valueBuilder.setTags(HBaseZeroCopyByteString.wrap(kv.getTagsArray(),
1049                 kv.getTagsOffset(), kv.getTagsLength()));
1050           }
1051           columnBuilder.addQualifierValue(valueBuilder.build());
1052         }
1053       }
1054       builder.addColumnValue(columnBuilder.build());
1055     }
1056     Map<String, byte[]> attributes = increment.getAttributesMap();
1057     if (!attributes.isEmpty()) {
1058       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1059       for (Map.Entry<String, byte[]> attribute : attributes.entrySet()) {
1060         attributeBuilder.setName(attribute.getKey());
1061         attributeBuilder.setValue(HBaseZeroCopyByteString.wrap(attribute.getValue()));
1062         builder.addAttribute(attributeBuilder.build());
1063       }
1064     }
1065     return builder.build();
1066   }
1067 
1068   public static MutationProto toMutation(final MutationType type, final Mutation mutation)
1069     throws IOException {
1070     return toMutation(type, mutation, HConstants.NO_NONCE);
1071   }
1072 
1073   /**
1074    * Create a protocol buffer Mutate based on a client Mutation
1075    *
1076    * @param type
1077    * @param mutation
1078    * @return a protobuf'd Mutation
1079    * @throws IOException
1080    */
1081   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1082     final long nonce) throws IOException {
1083     return toMutation(type, mutation, MutationProto.newBuilder(), nonce);
1084   }
1085 
1086   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1087       MutationProto.Builder builder) throws IOException {
1088     return toMutation(type, mutation, builder, HConstants.NO_NONCE);
1089   }
1090 
1091   public static MutationProto toMutation(final MutationType type, final Mutation mutation,
1092       MutationProto.Builder builder, long nonce)
1093   throws IOException {
1094     builder = getMutationBuilderAndSetCommonFields(type, mutation, builder);
1095     if (nonce != HConstants.NO_NONCE) {
1096       builder.setNonce(nonce);
1097     }
1098     ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
1099     QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
1100     for (Map.Entry<byte[],List<Cell>> family: mutation.getFamilyCellMap().entrySet()) {
1101       columnBuilder.clear();
1102       columnBuilder.setFamily(HBaseZeroCopyByteString.wrap(family.getKey()));
1103       for (Cell cell: family.getValue()) {
1104         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
1105         valueBuilder.setQualifier(HBaseZeroCopyByteString.wrap(
1106             kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()));
1107         valueBuilder.setValue(HBaseZeroCopyByteString.wrap(
1108             kv.getValueArray(), kv.getValueOffset(), kv.getValueLength()));
1109         valueBuilder.setTimestamp(kv.getTimestamp());
1110         if(cell.getTagsLength() > 0) {
1111           valueBuilder.setTags(ByteString.copyFrom(CellUtil.getTagArray(kv)));
1112         }
1113         if (type == MutationType.DELETE) {
1114           KeyValue.Type keyValueType = KeyValue.Type.codeToType(kv.getType());
1115           valueBuilder.setDeleteType(toDeleteType(keyValueType));
1116         }
1117         columnBuilder.addQualifierValue(valueBuilder.build());
1118       }
1119       builder.addColumnValue(columnBuilder.build());
1120     }
1121     return builder.build();
1122   }
1123 
1124   /**
1125    * Create a protocol buffer MutationProto based on a client Mutation. Does NOT include data.
1126    * Understanding is that the Cell will be transported other than via protobuf.
1127    * @param type
1128    * @param mutation
1129    * @param builder
1130    * @return a protobuf'd Mutation
1131    * @throws IOException
1132    */
1133   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1134       final MutationProto.Builder builder)  throws IOException {
1135     return toMutationNoData(type, mutation, builder, HConstants.NO_NONCE);
1136   }
1137 
1138   /**
1139    * Create a protocol buffer MutationProto based on a client Mutation.  Does NOT include data.
1140    * Understanding is that the Cell will be transported other than via protobuf.
1141    * @param type
1142    * @param mutation
1143    * @return a protobuf'd Mutation
1144    * @throws IOException
1145    */
1146   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation)
1147   throws IOException {
1148     MutationProto.Builder builder =  MutationProto.newBuilder();
1149     return toMutationNoData(type, mutation, builder);
1150   }
1151 
1152   public static MutationProto toMutationNoData(final MutationType type, final Mutation mutation,
1153       final MutationProto.Builder builder, long nonce) throws IOException {
1154     getMutationBuilderAndSetCommonFields(type, mutation, builder);
1155     builder.setAssociatedCellCount(mutation.size());
1156     if (nonce != HConstants.NO_NONCE) {
1157       builder.setNonce(nonce);
1158     }
1159     return builder.build();
1160   }
1161 
1162   /**
1163    * Code shared by {@link #toMutation(MutationType, Mutation)} and
1164    * {@link #toMutationNoData(MutationType, Mutation)}
1165    * @param type
1166    * @param mutation
1167    * @return A partly-filled out protobuf'd Mutation.
1168    */
1169   private static MutationProto.Builder getMutationBuilderAndSetCommonFields(final MutationType type,
1170       final Mutation mutation, MutationProto.Builder builder) {
1171     builder.setRow(HBaseZeroCopyByteString.wrap(mutation.getRow()));
1172     builder.setMutateType(type);
1173     builder.setDurability(toDurability(mutation.getDurability()));
1174     builder.setTimestamp(mutation.getTimeStamp());
1175     Map<String, byte[]> attributes = mutation.getAttributesMap();
1176     if (!attributes.isEmpty()) {
1177       NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
1178       for (Map.Entry<String, byte[]> attribute: attributes.entrySet()) {
1179         attributeBuilder.setName(attribute.getKey());
1180         attributeBuilder.setValue(HBaseZeroCopyByteString.wrap(attribute.getValue()));
1181         builder.addAttribute(attributeBuilder.build());
1182       }
1183     }
1184     return builder;
1185   }
1186 
1187   /**
1188    * Convert a client Result to a protocol buffer Result
1189    *
1190    * @param result the client Result to convert
1191    * @return the converted protocol buffer Result
1192    */
1193   public static ClientProtos.Result toResult(final Result result) {
1194     if (result.getExists() != null) {
1195       return toResult(result.getExists());
1196     }
1197 
1198     Cell[] cells = result.rawCells();
1199     if (cells == null || cells.length == 0) {
1200       return EMPTY_RESULT_PB;
1201     }
1202 
1203     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1204     for (Cell c : cells) {
1205       builder.addCell(toCell(c));
1206     }
1207 
1208     return builder.build();
1209   }
1210 
1211   /**
1212    * Convert a client Result to a protocol buffer Result
1213    *
1214    * @param existence the client existence to send
1215    * @return the converted protocol buffer Result
1216    */
1217   public static ClientProtos.Result toResult(final boolean existence) {
1218     return existence ? EMPTY_RESULT_PB_EXISTS_TRUE : EMPTY_RESULT_PB_EXISTS_FALSE;
1219   }
1220 
1221   /**
1222    * Convert a client Result to a protocol buffer Result.
1223    * The pb Result does not include the Cell data.  That is for transport otherwise.
1224    *
1225    * @param result the client Result to convert
1226    * @return the converted protocol buffer Result
1227    */
1228   public static ClientProtos.Result toResultNoData(final Result result) {
1229     if (result.getExists() != null) return toResult(result.getExists());
1230     int size = result.size();
1231     if (size == 0) return EMPTY_RESULT_PB;
1232     ClientProtos.Result.Builder builder = ClientProtos.Result.newBuilder();
1233     builder.setAssociatedCellCount(size);
1234     return builder.build();
1235   }
1236 
1237   /**
1238    * Convert a protocol buffer Result to a client Result
1239    *
1240    * @param proto the protocol buffer Result to convert
1241    * @return the converted client Result
1242    */
1243   public static Result toResult(final ClientProtos.Result proto) {
1244     if (proto.hasExists()) {
1245       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1246     }
1247 
1248     List<CellProtos.Cell> values = proto.getCellList();
1249     if (values.isEmpty()){
1250       return EMPTY_RESULT;
1251     }
1252 
1253     List<Cell> cells = new ArrayList<Cell>(values.size());
1254     for (CellProtos.Cell c : values) {
1255       cells.add(toCell(c));
1256     }
1257     return Result.create(cells, null);
1258   }
1259 
1260   /**
1261    * Convert a protocol buffer Result to a client Result
1262    *
1263    * @param proto the protocol buffer Result to convert
1264    * @param scanner Optional cell scanner.
1265    * @return the converted client Result
1266    * @throws IOException
1267    */
1268   public static Result toResult(final ClientProtos.Result proto, final CellScanner scanner)
1269   throws IOException {
1270     List<CellProtos.Cell> values = proto.getCellList();
1271 
1272     if (proto.hasExists()) {
1273       if ((values != null && !values.isEmpty()) ||
1274           (proto.hasAssociatedCellCount() && proto.getAssociatedCellCount() > 0)) {
1275         throw new IllegalArgumentException("bad proto: exists with cells is no allowed " + proto);
1276       }
1277       return proto.getExists() ? EMPTY_RESULT_EXISTS_TRUE : EMPTY_RESULT_EXISTS_FALSE;
1278     }
1279 
1280     // TODO: Unit test that has some Cells in scanner and some in the proto.
1281     List<Cell> cells = null;
1282     if (proto.hasAssociatedCellCount()) {
1283       int count = proto.getAssociatedCellCount();
1284       cells = new ArrayList<Cell>(count + values.size());
1285       for (int i = 0; i < count; i++) {
1286         if (!scanner.advance()) throw new IOException("Failed get " + i + " of " + count);
1287         cells.add(scanner.current());
1288       }
1289     }
1290 
1291     if (!values.isEmpty()){
1292       if (cells == null) cells = new ArrayList<Cell>(values.size());
1293       for (CellProtos.Cell c: values) {
1294         cells.add(toCell(c));
1295       }
1296     }
1297 
1298     return (cells == null || cells.isEmpty()) ? EMPTY_RESULT : Result.create(cells, null);
1299   }
1300 
1301 
1302   /**
1303    * Convert a ByteArrayComparable to a protocol buffer Comparator
1304    *
1305    * @param comparator the ByteArrayComparable to convert
1306    * @return the converted protocol buffer Comparator
1307    */
1308   public static ComparatorProtos.Comparator toComparator(ByteArrayComparable comparator) {
1309     ComparatorProtos.Comparator.Builder builder = ComparatorProtos.Comparator.newBuilder();
1310     builder.setName(comparator.getClass().getName());
1311     builder.setSerializedComparator(HBaseZeroCopyByteString.wrap(comparator.toByteArray()));
1312     return builder.build();
1313   }
1314 
1315   /**
1316    * Convert a protocol buffer Comparator to a ByteArrayComparable
1317    *
1318    * @param proto the protocol buffer Comparator to convert
1319    * @return the converted ByteArrayComparable
1320    */
1321   @SuppressWarnings("unchecked")
1322   public static ByteArrayComparable toComparator(ComparatorProtos.Comparator proto)
1323   throws IOException {
1324     String type = proto.getName();
1325     String funcName = "parseFrom";
1326     byte [] value = proto.getSerializedComparator().toByteArray();
1327     try {
1328       Class<? extends ByteArrayComparable> c =
1329         (Class<? extends ByteArrayComparable>)Class.forName(type, true, CLASS_LOADER);
1330       Method parseFrom = c.getMethod(funcName, byte[].class);
1331       if (parseFrom == null) {
1332         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1333       }
1334       return (ByteArrayComparable)parseFrom.invoke(null, value);
1335     } catch (Exception e) {
1336       throw new IOException(e);
1337     }
1338   }
1339 
1340   /**
1341    * Convert a protocol buffer Filter to a client Filter
1342    *
1343    * @param proto the protocol buffer Filter to convert
1344    * @return the converted Filter
1345    */
1346   @SuppressWarnings("unchecked")
1347   public static Filter toFilter(FilterProtos.Filter proto) throws IOException {
1348     String type = proto.getName();
1349     final byte [] value = proto.getSerializedFilter().toByteArray();
1350     String funcName = "parseFrom";
1351     try {
1352       Class<? extends Filter> c =
1353         (Class<? extends Filter>)Class.forName(type, true, CLASS_LOADER);
1354       Method parseFrom = c.getMethod(funcName, byte[].class);
1355       if (parseFrom == null) {
1356         throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
1357       }
1358       return (Filter)parseFrom.invoke(c, value);
1359     } catch (Exception e) {
1360       throw new IOException(e);
1361     }
1362   }
1363 
1364   /**
1365    * Convert a client Filter to a protocol buffer Filter
1366    *
1367    * @param filter the Filter to convert
1368    * @return the converted protocol buffer Filter
1369    */
1370   public static FilterProtos.Filter toFilter(Filter filter) throws IOException {
1371     FilterProtos.Filter.Builder builder = FilterProtos.Filter.newBuilder();
1372     builder.setName(filter.getClass().getName());
1373     builder.setSerializedFilter(HBaseZeroCopyByteString.wrap(filter.toByteArray()));
1374     return builder.build();
1375   }
1376 
1377   /**
1378    * Convert a delete KeyValue type to protocol buffer DeleteType.
1379    *
1380    * @param type
1381    * @return protocol buffer DeleteType
1382    * @throws IOException
1383    */
1384   public static DeleteType toDeleteType(
1385       KeyValue.Type type) throws IOException {
1386     switch (type) {
1387     case Delete:
1388       return DeleteType.DELETE_ONE_VERSION;
1389     case DeleteColumn:
1390       return DeleteType.DELETE_MULTIPLE_VERSIONS;
1391     case DeleteFamily:
1392       return DeleteType.DELETE_FAMILY;
1393     case DeleteFamilyVersion:
1394       return DeleteType.DELETE_FAMILY_VERSION;
1395     default:
1396         throw new IOException("Unknown delete type: " + type);
1397     }
1398   }
1399 
1400   /**
1401    * Convert a stringified protocol buffer exception Parameter to a Java Exception
1402    *
1403    * @param parameter the protocol buffer Parameter to convert
1404    * @return the converted Exception
1405    * @throws IOException if failed to deserialize the parameter
1406    */
1407   @SuppressWarnings("unchecked")
1408   public static Throwable toException(final NameBytesPair parameter) throws IOException {
1409     if (parameter == null || !parameter.hasValue()) return null;
1410     String desc = parameter.getValue().toStringUtf8();
1411     String type = parameter.getName();
1412     try {
1413       Class<? extends Throwable> c =
1414         (Class<? extends Throwable>)Class.forName(type, true, CLASS_LOADER);
1415       Constructor<? extends Throwable> cn = null;
1416       try {
1417         cn = c.getDeclaredConstructor(String.class);
1418         return cn.newInstance(desc);
1419       } catch (NoSuchMethodException e) {
1420         // Could be a raw RemoteException. See HBASE-8987.
1421         cn = c.getDeclaredConstructor(String.class, String.class);
1422         return cn.newInstance(type, desc);
1423       }
1424     } catch (Exception e) {
1425       throw new IOException(e);
1426     }
1427   }
1428 
1429 // Start helpers for Client
1430 
1431   /**
1432    * A helper to get a row of the closet one before using client protocol.
1433    *
1434    * @param client
1435    * @param regionName
1436    * @param row
1437    * @param family
1438    * @return the row or the closestRowBefore if it doesn't exist
1439    * @throws IOException
1440    * @deprecated since 0.99 - use reversed scanner instead.
1441    */
1442   @Deprecated
1443   public static Result getRowOrBefore(final ClientService.BlockingInterface client,
1444       final byte[] regionName, final byte[] row,
1445       final byte[] family) throws IOException {
1446     GetRequest request =
1447       RequestConverter.buildGetRowOrBeforeRequest(
1448         regionName, row, family);
1449     try {
1450       GetResponse response = client.get(null, request);
1451       if (!response.hasResult()) return null;
1452       return toResult(response.getResult());
1453     } catch (ServiceException se) {
1454       throw getRemoteException(se);
1455     }
1456   }
1457 
1458   /**
1459    * A helper to bulk load a list of HFiles using client protocol.
1460    *
1461    * @param client
1462    * @param familyPaths
1463    * @param regionName
1464    * @param assignSeqNum
1465    * @return true if all are loaded
1466    * @throws IOException
1467    */
1468   public static boolean bulkLoadHFile(final ClientService.BlockingInterface client,
1469       final List<Pair<byte[], String>> familyPaths,
1470       final byte[] regionName, boolean assignSeqNum) throws IOException {
1471     BulkLoadHFileRequest request =
1472       RequestConverter.buildBulkLoadHFileRequest(familyPaths, regionName, assignSeqNum);
1473     try {
1474       BulkLoadHFileResponse response =
1475         client.bulkLoadHFile(null, request);
1476       return response.getLoaded();
1477     } catch (ServiceException se) {
1478       throw getRemoteException(se);
1479     }
1480   }
1481 
1482   public static CoprocessorServiceResponse execService(final ClientService.BlockingInterface client,
1483       final CoprocessorServiceCall call, final byte[] regionName) throws IOException {
1484     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1485         .setCall(call).setRegion(
1486             RequestConverter.buildRegionSpecifier(REGION_NAME, regionName)).build();
1487     try {
1488       CoprocessorServiceResponse response =
1489           client.execService(null, request);
1490       return response;
1491     } catch (ServiceException se) {
1492       throw getRemoteException(se);
1493     }
1494   }
1495 
1496   public static CoprocessorServiceResponse execService(
1497     final MasterService.BlockingInterface client, final CoprocessorServiceCall call)
1498   throws IOException {
1499     CoprocessorServiceRequest request = CoprocessorServiceRequest.newBuilder()
1500         .setCall(call).setRegion(
1501             RequestConverter.buildRegionSpecifier(REGION_NAME, HConstants.EMPTY_BYTE_ARRAY)).build();
1502     try {
1503       CoprocessorServiceResponse response =
1504           client.execMasterService(null, request);
1505       return response;
1506     } catch (ServiceException se) {
1507       throw getRemoteException(se);
1508     }
1509   }
1510 
1511   @SuppressWarnings("unchecked")
1512   public static <T extends Service> T newServiceStub(Class<T> service, RpcChannel channel)
1513       throws Exception {
1514     return (T)Methods.call(service, null, "newStub",
1515         new Class[]{ RpcChannel.class }, new Object[]{ channel });
1516   }
1517 
1518 // End helpers for Client
1519 // Start helpers for Admin
1520 
1521   /**
1522    * A helper to retrieve region info given a region name
1523    * using admin protocol.
1524    *
1525    * @param admin
1526    * @param regionName
1527    * @return the retrieved region info
1528    * @throws IOException
1529    */
1530   public static HRegionInfo getRegionInfo(final AdminService.BlockingInterface admin,
1531       final byte[] regionName) throws IOException {
1532     try {
1533       GetRegionInfoRequest request =
1534         RequestConverter.buildGetRegionInfoRequest(regionName);
1535       GetRegionInfoResponse response =
1536         admin.getRegionInfo(null, request);
1537       return HRegionInfo.convert(response.getRegionInfo());
1538     } catch (ServiceException se) {
1539       throw getRemoteException(se);
1540     }
1541   }
1542 
1543   /**
1544    * A helper to close a region given a region name
1545    * using admin protocol.
1546    *
1547    * @param admin
1548    * @param regionName
1549    * @param transitionInZK
1550    * @throws IOException
1551    */
1552   public static void closeRegion(final AdminService.BlockingInterface admin,
1553       final ServerName server, final byte[] regionName, final boolean transitionInZK) throws IOException {
1554     CloseRegionRequest closeRegionRequest =
1555       RequestConverter.buildCloseRegionRequest(server, regionName, transitionInZK);
1556     try {
1557       admin.closeRegion(null, closeRegionRequest);
1558     } catch (ServiceException se) {
1559       throw getRemoteException(se);
1560     }
1561   }
1562 
1563   /**
1564    * A helper to close a region given a region name
1565    * using admin protocol.
1566    *
1567    * @param admin
1568    * @param regionName
1569    * @param versionOfClosingNode
1570    * @return true if the region is closed
1571    * @throws IOException
1572    */
1573   public static boolean closeRegion(final AdminService.BlockingInterface admin,
1574       final ServerName server,
1575       final byte[] regionName,
1576       final int versionOfClosingNode, final ServerName destinationServer,
1577       final boolean transitionInZK) throws IOException {
1578     CloseRegionRequest closeRegionRequest =
1579       RequestConverter.buildCloseRegionRequest(server,
1580         regionName, versionOfClosingNode, destinationServer, transitionInZK);
1581     try {
1582       CloseRegionResponse response = admin.closeRegion(null, closeRegionRequest);
1583       return ResponseConverter.isClosed(response);
1584     } catch (ServiceException se) {
1585       throw getRemoteException(se);
1586     }
1587   }
1588 
1589 
1590   /**
1591    * A helper to open a region using admin protocol.
1592    * @param admin
1593    * @param region
1594    * @throws IOException
1595    */
1596   public static void openRegion(final AdminService.BlockingInterface admin,
1597       ServerName server, final HRegionInfo region) throws IOException {
1598     OpenRegionRequest request =
1599       RequestConverter.buildOpenRegionRequest(server, region, -1, null, null);
1600     try {
1601       admin.openRegion(null, request);
1602     } catch (ServiceException se) {
1603       throw ProtobufUtil.getRemoteException(se);
1604     }
1605   }
1606 
1607   /**
1608    * A helper to get the all the online regions on a region
1609    * server using admin protocol.
1610    *
1611    * @param admin
1612    * @return a list of online region info
1613    * @throws IOException
1614    */
1615   public static List<HRegionInfo> getOnlineRegions(final AdminService.BlockingInterface admin)
1616   throws IOException {
1617     GetOnlineRegionRequest request = RequestConverter.buildGetOnlineRegionRequest();
1618     GetOnlineRegionResponse response = null;
1619     try {
1620       response = admin.getOnlineRegion(null, request);
1621     } catch (ServiceException se) {
1622       throw getRemoteException(se);
1623     }
1624     return getRegionInfos(response);
1625   }
1626 
1627   /**
1628    * Get the list of region info from a GetOnlineRegionResponse
1629    *
1630    * @param proto the GetOnlineRegionResponse
1631    * @return the list of region info or null if <code>proto</code> is null
1632    */
1633   static List<HRegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
1634     if (proto == null) return null;
1635     List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
1636     for (RegionInfo regionInfo: proto.getRegionInfoList()) {
1637       regionInfos.add(HRegionInfo.convert(regionInfo));
1638     }
1639     return regionInfos;
1640   }
1641 
1642   /**
1643    * A helper to get the info of a region server using admin protocol.
1644    *
1645    * @param admin
1646    * @return the server name
1647    * @throws IOException
1648    */
1649   public static ServerInfo getServerInfo(final AdminService.BlockingInterface admin)
1650   throws IOException {
1651     GetServerInfoRequest request = RequestConverter.buildGetServerInfoRequest();
1652     try {
1653       GetServerInfoResponse response = admin.getServerInfo(null, request);
1654       return response.getServerInfo();
1655     } catch (ServiceException se) {
1656       throw getRemoteException(se);
1657     }
1658   }
1659 
1660   /**
1661    * A helper to get the list of files of a column family
1662    * on a given region using admin protocol.
1663    *
1664    * @param admin
1665    * @param regionName
1666    * @param family
1667    * @return the list of store files
1668    * @throws IOException
1669    */
1670   public static List<String> getStoreFiles(final AdminService.BlockingInterface admin,
1671       final byte[] regionName, final byte[] family)
1672   throws IOException {
1673     GetStoreFileRequest request =
1674       RequestConverter.buildGetStoreFileRequest(regionName, family);
1675     try {
1676       GetStoreFileResponse response = admin.getStoreFile(null, request);
1677       return response.getStoreFileList();
1678     } catch (ServiceException se) {
1679       throw ProtobufUtil.getRemoteException(se);
1680     }
1681   }
1682 
1683   /**
1684    * A helper to split a region using admin protocol.
1685    *
1686    * @param admin
1687    * @param hri
1688    * @param splitPoint
1689    * @throws IOException
1690    */
1691   public static void split(final AdminService.BlockingInterface admin,
1692       final HRegionInfo hri, byte[] splitPoint) throws IOException {
1693     SplitRegionRequest request =
1694       RequestConverter.buildSplitRegionRequest(hri.getRegionName(), splitPoint);
1695     try {
1696       admin.splitRegion(null, request);
1697     } catch (ServiceException se) {
1698       throw ProtobufUtil.getRemoteException(se);
1699     }
1700   }
1701 
1702   /**
1703    * A helper to merge regions using admin protocol. Send request to
1704    * regionserver.
1705    * @param admin
1706    * @param region_a
1707    * @param region_b
1708    * @param forcible true if do a compulsory merge, otherwise we will only merge
1709    *          two adjacent regions
1710    * @throws IOException
1711    */
1712   public static void mergeRegions(final AdminService.BlockingInterface admin,
1713       final HRegionInfo region_a, final HRegionInfo region_b,
1714       final boolean forcible) throws IOException {
1715     MergeRegionsRequest request = RequestConverter.buildMergeRegionsRequest(
1716         region_a.getRegionName(), region_b.getRegionName(),forcible);
1717     try {
1718       admin.mergeRegions(null, request);
1719     } catch (ServiceException se) {
1720       throw ProtobufUtil.getRemoteException(se);
1721     }
1722   }
1723 
1724 // End helpers for Admin
1725 
1726   /*
1727    * Get the total (read + write) requests from a RegionLoad pb
1728    * @param rl - RegionLoad pb
1729    * @return total (read + write) requests
1730    */
1731   public static long getTotalRequestsCount(RegionLoad rl) {
1732     if (rl == null) {
1733       return 0;
1734     }
1735 
1736     return rl.getReadRequestsCount() + rl.getWriteRequestsCount();
1737   }
1738 
1739 
1740   /**
1741    * @param m Message to get delimited pb serialization of (with pb magic prefix)
1742    */
1743   public static byte [] toDelimitedByteArray(final Message m) throws IOException {
1744     // Allocate arbitrary big size so we avoid resizing.
1745     ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
1746     baos.write(PB_MAGIC);
1747     m.writeDelimitedTo(baos);
1748     return baos.toByteArray();
1749   }
1750 
1751   /**
1752    * Converts a Permission proto to a client Permission object.
1753    *
1754    * @param proto the protobuf Permission
1755    * @return the converted Permission
1756    */
1757   public static Permission toPermission(AccessControlProtos.Permission proto) {
1758     if (proto.getType() != AccessControlProtos.Permission.Type.Global) {
1759       return toTablePermission(proto);
1760     } else {
1761       List<Permission.Action> actions = toPermissionActions(proto.getGlobalPermission().getActionList());
1762       return new Permission(actions.toArray(new Permission.Action[actions.size()]));
1763     }
1764   }
1765 
1766   /**
1767    * Converts a Permission proto to a client TablePermission object.
1768    *
1769    * @param proto the protobuf Permission
1770    * @return the converted TablePermission
1771    */
1772   public static TablePermission toTablePermission(AccessControlProtos.Permission proto) {
1773     if(proto.getType() == AccessControlProtos.Permission.Type.Global) {
1774       AccessControlProtos.GlobalPermission perm = proto.getGlobalPermission();
1775       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1776 
1777       return new TablePermission(null, null, null,
1778           actions.toArray(new Permission.Action[actions.size()]));
1779     }
1780     if(proto.getType() == AccessControlProtos.Permission.Type.Namespace) {
1781       AccessControlProtos.NamespacePermission perm = proto.getNamespacePermission();
1782       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1783 
1784       if(!proto.hasNamespacePermission()) {
1785         throw new IllegalStateException("Namespace must not be empty in NamespacePermission");
1786       }
1787       String namespace = perm.getNamespaceName().toStringUtf8();
1788       return new TablePermission(namespace, actions.toArray(new Permission.Action[actions.size()]));
1789     }
1790     if(proto.getType() == AccessControlProtos.Permission.Type.Table) {
1791       AccessControlProtos.TablePermission perm = proto.getTablePermission();
1792       List<Permission.Action> actions = toPermissionActions(perm.getActionList());
1793 
1794       byte[] qualifier = null;
1795       byte[] family = null;
1796       TableName table = null;
1797 
1798       if (!perm.hasTableName()) {
1799         throw new IllegalStateException("TableName cannot be empty");
1800       }
1801       table = ProtobufUtil.toTableName(perm.getTableName());
1802 
1803       if (perm.hasFamily()) family = perm.getFamily().toByteArray();
1804       if (perm.hasQualifier()) qualifier = perm.getQualifier().toByteArray();
1805 
1806       return new TablePermission(table, family, qualifier,
1807           actions.toArray(new Permission.Action[actions.size()]));
1808     }
1809     throw new IllegalStateException("Unrecognize Perm Type: "+proto.getType());
1810   }
1811 
1812   /**
1813    * Convert a client Permission to a Permission proto
1814    *
1815    * @param perm the client Permission
1816    * @return the protobuf Permission
1817    */
1818   public static AccessControlProtos.Permission toPermission(Permission perm) {
1819     AccessControlProtos.Permission.Builder ret = AccessControlProtos.Permission.newBuilder();
1820     if (perm instanceof TablePermission) {
1821       TablePermission tablePerm = (TablePermission)perm;
1822       if(tablePerm.hasNamespace()) {
1823         ret.setType(AccessControlProtos.Permission.Type.Namespace);
1824 
1825         AccessControlProtos.NamespacePermission.Builder builder =
1826             AccessControlProtos.NamespacePermission.newBuilder();
1827         builder.setNamespaceName(ByteString.copyFromUtf8(tablePerm.getNamespace()));
1828         Permission.Action actions[] = perm.getActions();
1829         if (actions != null) {
1830           for (Permission.Action a : actions) {
1831             builder.addAction(toPermissionAction(a));
1832           }
1833 	}
1834         ret.setNamespacePermission(builder);
1835         return ret.build();
1836       } else if (tablePerm.hasTable()) {
1837         ret.setType(AccessControlProtos.Permission.Type.Table);
1838 
1839         AccessControlProtos.TablePermission.Builder builder =
1840             AccessControlProtos.TablePermission.newBuilder();
1841         builder.setTableName(ProtobufUtil.toProtoTableName(tablePerm.getTableName()));
1842         if (tablePerm.hasFamily()) {
1843           builder.setFamily(HBaseZeroCopyByteString.wrap(tablePerm.getFamily()));
1844         }
1845         if (tablePerm.hasQualifier()) {
1846           builder.setQualifier(HBaseZeroCopyByteString.wrap(tablePerm.getQualifier()));
1847         }
1848         Permission.Action actions[] = perm.getActions();
1849         if (actions != null) {
1850           for (Permission.Action a : actions) {
1851             builder.addAction(toPermissionAction(a));
1852           }
1853         }
1854         ret.setTablePermission(builder);
1855         return ret.build();
1856       }
1857     }
1858 
1859     ret.setType(AccessControlProtos.Permission.Type.Global);
1860 
1861     AccessControlProtos.GlobalPermission.Builder builder =
1862         AccessControlProtos.GlobalPermission.newBuilder();
1863     Permission.Action actions[] = perm.getActions();
1864     if (actions != null) {
1865       for (Permission.Action a: actions) {
1866         builder.addAction(toPermissionAction(a));
1867       }
1868     }
1869     ret.setGlobalPermission(builder);
1870     return ret.build();
1871   }
1872 
1873   /**
1874    * Converts a list of Permission.Action proto to a list of client Permission.Action objects.
1875    *
1876    * @param protoActions the list of protobuf Actions
1877    * @return the converted list of Actions
1878    */
1879   public static List<Permission.Action> toPermissionActions(
1880       List<AccessControlProtos.Permission.Action> protoActions) {
1881     List<Permission.Action> actions = new ArrayList<Permission.Action>(protoActions.size());
1882     for (AccessControlProtos.Permission.Action a : protoActions) {
1883       actions.add(toPermissionAction(a));
1884     }
1885     return actions;
1886   }
1887 
1888   /**
1889    * Converts a Permission.Action proto to a client Permission.Action object.
1890    *
1891    * @param action the protobuf Action
1892    * @return the converted Action
1893    */
1894   public static Permission.Action toPermissionAction(
1895       AccessControlProtos.Permission.Action action) {
1896     switch (action) {
1897       case READ:
1898         return Permission.Action.READ;
1899       case WRITE:
1900         return Permission.Action.WRITE;
1901       case EXEC:
1902         return Permission.Action.EXEC;
1903       case CREATE:
1904         return Permission.Action.CREATE;
1905       case ADMIN:
1906         return Permission.Action.ADMIN;
1907     }
1908     throw new IllegalArgumentException("Unknown action value "+action.name());
1909   }
1910 
1911   /**
1912    * Convert a client Permission.Action to a Permission.Action proto
1913    *
1914    * @param action the client Action
1915    * @return the protobuf Action
1916    */
1917   public static AccessControlProtos.Permission.Action toPermissionAction(
1918       Permission.Action action) {
1919     switch (action) {
1920       case READ:
1921         return AccessControlProtos.Permission.Action.READ;
1922       case WRITE:
1923         return AccessControlProtos.Permission.Action.WRITE;
1924       case EXEC:
1925         return AccessControlProtos.Permission.Action.EXEC;
1926       case CREATE:
1927         return AccessControlProtos.Permission.Action.CREATE;
1928       case ADMIN:
1929         return AccessControlProtos.Permission.Action.ADMIN;
1930     }
1931     throw new IllegalArgumentException("Unknown action value "+action.name());
1932   }
1933 
1934   /**
1935    * Convert a client user permission to a user permission proto
1936    *
1937    * @param perm the client UserPermission
1938    * @return the protobuf UserPermission
1939    */
1940   public static AccessControlProtos.UserPermission toUserPermission(UserPermission perm) {
1941     return AccessControlProtos.UserPermission.newBuilder()
1942         .setUser(HBaseZeroCopyByteString.wrap(perm.getUser()))
1943         .setPermission(toPermission(perm))
1944         .build();
1945   }
1946 
1947   /**
1948    * Converts a user permission proto to a client user permission object.
1949    *
1950    * @param proto the protobuf UserPermission
1951    * @return the converted UserPermission
1952    */
1953   public static UserPermission toUserPermission(AccessControlProtos.UserPermission proto) {
1954     return new UserPermission(proto.getUser().toByteArray(),
1955         toTablePermission(proto.getPermission()));
1956   }
1957 
1958   /**
1959    * Convert a ListMultimap<String, TablePermission> where key is username
1960    * to a protobuf UserPermission
1961    *
1962    * @param perm the list of user and table permissions
1963    * @return the protobuf UserTablePermissions
1964    */
1965   public static AccessControlProtos.UsersAndPermissions toUserTablePermissions(
1966       ListMultimap<String, TablePermission> perm) {
1967     AccessControlProtos.UsersAndPermissions.Builder builder =
1968                   AccessControlProtos.UsersAndPermissions.newBuilder();
1969     for (Map.Entry<String, Collection<TablePermission>> entry : perm.asMap().entrySet()) {
1970       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
1971                   AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
1972       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
1973       for (TablePermission tablePerm: entry.getValue()) {
1974         userPermBuilder.addPermissions(toPermission(tablePerm));
1975       }
1976       builder.addUserPermissions(userPermBuilder.build());
1977     }
1978     return builder.build();
1979   }
1980 
1981   /**
1982    * A utility used to grant a user global permissions.
1983    * <p>
1984    * It's also called by the shell, in case you want to find references.
1985    *
1986    * @param protocol the AccessControlService protocol proxy
1987    * @param userShortName the short name of the user to grant permissions
1988    * @param actions the permissions to be granted
1989    * @throws ServiceException
1990    */
1991   public static void grant(AccessControlService.BlockingInterface protocol,
1992       String userShortName, Permission.Action... actions) throws ServiceException {
1993     List<AccessControlProtos.Permission.Action> permActions =
1994         Lists.newArrayListWithCapacity(actions.length);
1995     for (Permission.Action a : actions) {
1996       permActions.add(ProtobufUtil.toPermissionAction(a));
1997     }
1998     AccessControlProtos.GrantRequest request = RequestConverter.
1999       buildGrantRequest(userShortName, permActions.toArray(
2000         new AccessControlProtos.Permission.Action[actions.length]));
2001     protocol.grant(null, request);
2002   }
2003 
2004   /**
2005    * A utility used to grant a user table permissions. The permissions will
2006    * be for a table table/column family/qualifier.
2007    * <p>
2008    * It's also called by the shell, in case you want to find references.
2009    *
2010    * @param protocol the AccessControlService protocol proxy
2011    * @param userShortName the short name of the user to grant permissions
2012    * @param tableName optional table name
2013    * @param f optional column family
2014    * @param q optional qualifier
2015    * @param actions the permissions to be granted
2016    * @throws ServiceException
2017    */
2018   public static void grant(AccessControlService.BlockingInterface protocol,
2019       String userShortName, TableName tableName, byte[] f, byte[] q,
2020       Permission.Action... actions) throws ServiceException {
2021     List<AccessControlProtos.Permission.Action> permActions =
2022         Lists.newArrayListWithCapacity(actions.length);
2023     for (Permission.Action a : actions) {
2024       permActions.add(ProtobufUtil.toPermissionAction(a));
2025     }
2026     AccessControlProtos.GrantRequest request = RequestConverter.
2027       buildGrantRequest(userShortName, tableName, f, q, permActions.toArray(
2028         new AccessControlProtos.Permission.Action[actions.length]));
2029     protocol.grant(null, request);
2030   }
2031 
2032   /**
2033    * A utility used to grant a user namespace permissions.
2034    * <p>
2035    * It's also called by the shell, in case you want to find references.
2036    *
2037    * @param protocol the AccessControlService protocol proxy
2038    * @param namespace the short name of the user to grant permissions
2039    * @param actions the permissions to be granted
2040    * @throws ServiceException
2041    */
2042   public static void grant(AccessControlService.BlockingInterface protocol,
2043       String userShortName, String namespace,
2044       Permission.Action... actions) throws ServiceException {
2045     List<AccessControlProtos.Permission.Action> permActions =
2046         Lists.newArrayListWithCapacity(actions.length);
2047     for (Permission.Action a : actions) {
2048       permActions.add(ProtobufUtil.toPermissionAction(a));
2049     }
2050     AccessControlProtos.GrantRequest request = RequestConverter.
2051       buildGrantRequest(userShortName, namespace, permActions.toArray(
2052         new AccessControlProtos.Permission.Action[actions.length]));
2053     protocol.grant(null, request);
2054   }
2055 
2056   /**
2057    * A utility used to revoke a user's global permissions.
2058    * <p>
2059    * It's also called by the shell, in case you want to find references.
2060    *
2061    * @param protocol the AccessControlService protocol proxy
2062    * @param userShortName the short name of the user to revoke permissions
2063    * @param actions the permissions to be revoked
2064    * @throws ServiceException
2065    */
2066   public static void revoke(AccessControlService.BlockingInterface protocol,
2067       String userShortName, Permission.Action... actions) throws ServiceException {
2068     List<AccessControlProtos.Permission.Action> permActions =
2069         Lists.newArrayListWithCapacity(actions.length);
2070     for (Permission.Action a : actions) {
2071       permActions.add(ProtobufUtil.toPermissionAction(a));
2072     }
2073     AccessControlProtos.RevokeRequest request = RequestConverter.
2074       buildRevokeRequest(userShortName, permActions.toArray(
2075         new AccessControlProtos.Permission.Action[actions.length]));
2076     protocol.revoke(null, request);
2077   }
2078 
2079   /**
2080    * A utility used to revoke a user's table permissions. The permissions will
2081    * be for a table/column family/qualifier.
2082    * <p>
2083    * It's also called by the shell, in case you want to find references.
2084    *
2085    * @param protocol the AccessControlService protocol proxy
2086    * @param userShortName the short name of the user to revoke permissions
2087    * @param tableName optional table name
2088    * @param f optional column family
2089    * @param q optional qualifier
2090    * @param actions the permissions to be revoked
2091    * @throws ServiceException
2092    */
2093   public static void revoke(AccessControlService.BlockingInterface protocol,
2094       String userShortName, TableName tableName, byte[] f, byte[] q,
2095       Permission.Action... actions) throws ServiceException {
2096     List<AccessControlProtos.Permission.Action> permActions =
2097         Lists.newArrayListWithCapacity(actions.length);
2098     for (Permission.Action a : actions) {
2099       permActions.add(ProtobufUtil.toPermissionAction(a));
2100     }
2101     AccessControlProtos.RevokeRequest request = RequestConverter.
2102       buildRevokeRequest(userShortName, tableName, f, q, permActions.toArray(
2103         new AccessControlProtos.Permission.Action[actions.length]));
2104     protocol.revoke(null, request);
2105   }
2106 
2107   /**
2108    * A utility used to revoke a user's namespace permissions.
2109    * <p>
2110    * It's also called by the shell, in case you want to find references.
2111    *
2112    * @param protocol the AccessControlService protocol proxy
2113    * @param userShortName the short name of the user to revoke permissions
2114    * @param namespace optional table name
2115    * @param actions the permissions to be revoked
2116    * @throws ServiceException
2117    */
2118   public static void revoke(AccessControlService.BlockingInterface protocol,
2119       String userShortName, String namespace,
2120       Permission.Action... actions) throws ServiceException {
2121     List<AccessControlProtos.Permission.Action> permActions =
2122         Lists.newArrayListWithCapacity(actions.length);
2123     for (Permission.Action a : actions) {
2124       permActions.add(ProtobufUtil.toPermissionAction(a));
2125     }
2126     AccessControlProtos.RevokeRequest request = RequestConverter.
2127       buildRevokeRequest(userShortName, namespace, permActions.toArray(
2128         new AccessControlProtos.Permission.Action[actions.length]));
2129     protocol.revoke(null, request);
2130   }
2131 
2132   /**
2133    * A utility used to get user's global permissions.
2134    * <p>
2135    * It's also called by the shell, in case you want to find references.
2136    *
2137    * @param protocol the AccessControlService protocol proxy
2138    * @throws ServiceException
2139    */
2140   public static List<UserPermission> getUserPermissions(
2141       AccessControlService.BlockingInterface protocol) throws ServiceException {
2142     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2143       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2144     builder.setType(AccessControlProtos.Permission.Type.Global);
2145     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2146     AccessControlProtos.GetUserPermissionsResponse response =
2147       protocol.getUserPermissions(null, request);
2148     List<UserPermission> perms = new ArrayList<UserPermission>();
2149     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2150       perms.add(ProtobufUtil.toUserPermission(perm));
2151     }
2152     return perms;
2153   }
2154 
2155   /**
2156    * A utility used to get user table permissions.
2157    * <p>
2158    * It's also called by the shell, in case you want to find references.
2159    *
2160    * @param protocol the AccessControlService protocol proxy
2161    * @param t optional table name
2162    * @throws ServiceException
2163    */
2164   public static List<UserPermission> getUserPermissions(
2165       AccessControlService.BlockingInterface protocol,
2166       TableName t) throws ServiceException {
2167     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2168       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2169     if (t != null) {
2170       builder.setTableName(ProtobufUtil.toProtoTableName(t));
2171     }
2172     builder.setType(AccessControlProtos.Permission.Type.Table);
2173     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2174     AccessControlProtos.GetUserPermissionsResponse response =
2175       protocol.getUserPermissions(null, request);
2176     List<UserPermission> perms = new ArrayList<UserPermission>();
2177     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2178       perms.add(ProtobufUtil.toUserPermission(perm));
2179     }
2180     return perms;
2181   }
2182 
2183   /**
2184    * A utility used to get permissions for selected namespace.
2185    * <p>
2186    * It's also called by the shell, in case you want to find references.
2187    *
2188    * @param protocol the AccessControlService protocol proxy
2189    * @param namespace name of the namespace
2190    * @throws ServiceException
2191    */
2192   public static List<UserPermission> getUserPermissions(
2193       AccessControlService.BlockingInterface protocol,
2194       byte[] namespace) throws ServiceException {
2195     AccessControlProtos.GetUserPermissionsRequest.Builder builder =
2196       AccessControlProtos.GetUserPermissionsRequest.newBuilder();
2197     if (namespace != null) {
2198       builder.setNamespaceName(HBaseZeroCopyByteString.wrap(namespace));
2199     }
2200     builder.setType(AccessControlProtos.Permission.Type.Namespace);
2201     AccessControlProtos.GetUserPermissionsRequest request = builder.build();
2202     AccessControlProtos.GetUserPermissionsResponse response =
2203       protocol.getUserPermissions(null, request);
2204     List<UserPermission> perms = new ArrayList<UserPermission>();
2205     for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) {
2206       perms.add(ProtobufUtil.toUserPermission(perm));
2207     }
2208     return perms;
2209   }
2210 
2211   /**
2212    * Convert a protobuf UserTablePermissions to a
2213    * ListMultimap<String, TablePermission> where key is username.
2214    *
2215    * @param proto the protobuf UserPermission
2216    * @return the converted UserPermission
2217    */
2218   public static ListMultimap<String, TablePermission> toUserTablePermissions(
2219       AccessControlProtos.UsersAndPermissions proto) {
2220     ListMultimap<String, TablePermission> perms = ArrayListMultimap.create();
2221     AccessControlProtos.UsersAndPermissions.UserPermissions userPerm;
2222 
2223     for (int i = 0; i < proto.getUserPermissionsCount(); i++) {
2224       userPerm = proto.getUserPermissions(i);
2225       for (int j = 0; j < userPerm.getPermissionsCount(); j++) {
2226         TablePermission tablePerm = toTablePermission(userPerm.getPermissions(j));
2227         perms.put(userPerm.getUser().toStringUtf8(), tablePerm);
2228       }
2229     }
2230 
2231     return perms;
2232   }
2233 
2234   /**
2235    * Converts a Token instance (with embedded identifier) to the protobuf representation.
2236    *
2237    * @param token the Token instance to copy
2238    * @return the protobuf Token message
2239    */
2240   public static AuthenticationProtos.Token toToken(Token<AuthenticationTokenIdentifier> token) {
2241     AuthenticationProtos.Token.Builder builder = AuthenticationProtos.Token.newBuilder();
2242     builder.setIdentifier(HBaseZeroCopyByteString.wrap(token.getIdentifier()));
2243     builder.setPassword(HBaseZeroCopyByteString.wrap(token.getPassword()));
2244     if (token.getService() != null) {
2245       builder.setService(ByteString.copyFromUtf8(token.getService().toString()));
2246     }
2247     return builder.build();
2248   }
2249 
2250   /**
2251    * Converts a protobuf Token message back into a Token instance.
2252    *
2253    * @param proto the protobuf Token message
2254    * @return the Token instance
2255    */
2256   public static Token<AuthenticationTokenIdentifier> toToken(AuthenticationProtos.Token proto) {
2257     return new Token<AuthenticationTokenIdentifier>(
2258         proto.hasIdentifier() ? proto.getIdentifier().toByteArray() : null,
2259         proto.hasPassword() ? proto.getPassword().toByteArray() : null,
2260         AuthenticationTokenIdentifier.AUTH_TOKEN_TYPE,
2261         proto.hasService() ? new Text(proto.getService().toStringUtf8()) : null);
2262   }
2263 
2264   /**
2265    * Find the HRegion encoded name based on a region specifier
2266    *
2267    * @param regionSpecifier the region specifier
2268    * @return the corresponding region's encoded name
2269    * @throws DoNotRetryIOException if the specifier type is unsupported
2270    */
2271   public static String getRegionEncodedName(
2272       final RegionSpecifier regionSpecifier) throws DoNotRetryIOException {
2273     byte[] value = regionSpecifier.getValue().toByteArray();
2274     RegionSpecifierType type = regionSpecifier.getType();
2275     switch (type) {
2276       case REGION_NAME:
2277         return HRegionInfo.encodeRegionName(value);
2278       case ENCODED_REGION_NAME:
2279         return Bytes.toString(value);
2280       default:
2281         throw new DoNotRetryIOException(
2282           "Unsupported region specifier type: " + type);
2283     }
2284   }
2285 
2286   public static ScanMetrics toScanMetrics(final byte[] bytes) {
2287     Parser<MapReduceProtos.ScanMetrics> parser = MapReduceProtos.ScanMetrics.PARSER;
2288     MapReduceProtos.ScanMetrics pScanMetrics = null;
2289     try {
2290       pScanMetrics = parser.parseFrom(bytes);
2291     } catch (InvalidProtocolBufferException e) {
2292       //Ignored there are just no key values to add.
2293     }
2294     ScanMetrics scanMetrics = new ScanMetrics();
2295     if (pScanMetrics != null) {
2296       for (HBaseProtos.NameInt64Pair pair : pScanMetrics.getMetricsList()) {
2297         if (pair.hasName() && pair.hasValue()) {
2298           scanMetrics.setCounter(pair.getName(), pair.getValue());
2299         }
2300       }
2301     }
2302     return scanMetrics;
2303   }
2304 
2305   public static MapReduceProtos.ScanMetrics toScanMetrics(ScanMetrics scanMetrics) {
2306     MapReduceProtos.ScanMetrics.Builder builder = MapReduceProtos.ScanMetrics.newBuilder();
2307     Map<String, Long> metrics = scanMetrics.getMetricsMap();
2308     for (Entry<String, Long> e : metrics.entrySet()) {
2309       HBaseProtos.NameInt64Pair nameInt64Pair =
2310           HBaseProtos.NameInt64Pair.newBuilder()
2311               .setName(e.getKey())
2312               .setValue(e.getValue())
2313               .build();
2314       builder.addMetrics(nameInt64Pair);
2315     }
2316     return builder.build();
2317   }
2318 
2319   /**
2320    * Unwraps an exception from a protobuf service into the underlying (expected) IOException.
2321    * This method will <strong>always</strong> throw an exception.
2322    * @param se the {@code ServiceException} instance to convert into an {@code IOException}
2323    */
2324   public static void toIOException(ServiceException se) throws IOException {
2325     if (se == null) {
2326       throw new NullPointerException("Null service exception passed!");
2327     }
2328 
2329     Throwable cause = se.getCause();
2330     if (cause != null && cause instanceof IOException) {
2331       throw (IOException)cause;
2332     }
2333     throw new IOException(se);
2334   }
2335 
2336   public static CellProtos.Cell toCell(final Cell kv) {
2337     // Doing this is going to kill us if we do it for all data passed.
2338     // St.Ack 20121205
2339     CellProtos.Cell.Builder kvbuilder = CellProtos.Cell.newBuilder();
2340     kvbuilder.setRow(HBaseZeroCopyByteString.wrap(kv.getRowArray(), kv.getRowOffset(),
2341         kv.getRowLength()));
2342     kvbuilder.setFamily(HBaseZeroCopyByteString.wrap(kv.getFamilyArray(),
2343         kv.getFamilyOffset(), kv.getFamilyLength()));
2344     kvbuilder.setQualifier(HBaseZeroCopyByteString.wrap(kv.getQualifierArray(),
2345         kv.getQualifierOffset(), kv.getQualifierLength()));
2346     kvbuilder.setCellType(CellProtos.CellType.valueOf(kv.getTypeByte()));
2347     kvbuilder.setTimestamp(kv.getTimestamp());
2348     kvbuilder.setValue(HBaseZeroCopyByteString.wrap(kv.getValueArray(), kv.getValueOffset(),
2349         kv.getValueLength()));
2350     return kvbuilder.build();
2351   }
2352 
2353   public static Cell toCell(final CellProtos.Cell cell) {
2354     // Doing this is going to kill us if we do it for all data passed.
2355     // St.Ack 20121205
2356     return CellUtil.createCell(cell.getRow().toByteArray(),
2357       cell.getFamily().toByteArray(),
2358       cell.getQualifier().toByteArray(),
2359       cell.getTimestamp(),
2360       (byte)cell.getCellType().getNumber(),
2361       cell.getValue().toByteArray());
2362   }
2363 
2364   public static HBaseProtos.NamespaceDescriptor toProtoNamespaceDescriptor(NamespaceDescriptor ns) {
2365     HBaseProtos.NamespaceDescriptor.Builder b =
2366         HBaseProtos.NamespaceDescriptor.newBuilder()
2367             .setName(ByteString.copyFromUtf8(ns.getName()));
2368     for(Map.Entry<String, String> entry: ns.getConfiguration().entrySet()) {
2369       b.addConfiguration(HBaseProtos.NameStringPair.newBuilder()
2370           .setName(entry.getKey())
2371           .setValue(entry.getValue()));
2372     }
2373     return b.build();
2374   }
2375 
2376   public static NamespaceDescriptor toNamespaceDescriptor(
2377       HBaseProtos.NamespaceDescriptor desc) throws IOException {
2378     NamespaceDescriptor.Builder b =
2379       NamespaceDescriptor.create(desc.getName().toStringUtf8());
2380     for(HBaseProtos.NameStringPair prop : desc.getConfigurationList()) {
2381       b.addConfiguration(prop.getName(), prop.getValue());
2382     }
2383     return b.build();
2384   }
2385 
2386   /**
2387    * Get an instance of the argument type declared in a class's signature. The
2388    * argument type is assumed to be a PB Message subclass, and the instance is
2389    * created using parseFrom method on the passed ByteString.
2390    * @param runtimeClass the runtime type of the class
2391    * @param position the position of the argument in the class declaration
2392    * @param b the ByteString which should be parsed to get the instance created
2393    * @return the instance
2394    * @throws IOException
2395    */
2396   @SuppressWarnings("unchecked")
2397   public static <T extends Message>
2398   T getParsedGenericInstance(Class<?> runtimeClass, int position, ByteString b)
2399       throws IOException {
2400     Type type = runtimeClass.getGenericSuperclass();
2401     Type argType = ((ParameterizedType)type).getActualTypeArguments()[position];
2402     Class<T> classType = (Class<T>)argType;
2403     T inst;
2404     try {
2405       Method m = classType.getMethod("parseFrom", ByteString.class);
2406       inst = (T)m.invoke(null, b);
2407       return inst;
2408     } catch (SecurityException e) {
2409       throw new IOException(e);
2410     } catch (NoSuchMethodException e) {
2411       throw new IOException(e);
2412     } catch (IllegalArgumentException e) {
2413       throw new IOException(e);
2414     } catch (InvocationTargetException e) {
2415       throw new IOException(e);
2416     } catch (IllegalAccessException e) {
2417       throw new IOException(e);
2418     }
2419   }
2420 
2421   public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family,
2422       List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
2423     // compaction descriptor contains relative paths.
2424     // input / output paths are relative to the store dir
2425     // store dir is relative to region dir
2426     CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder()
2427         .setTableName(HBaseZeroCopyByteString.wrap(info.getTableName()))
2428         .setEncodedRegionName(HBaseZeroCopyByteString.wrap(info.getEncodedNameAsBytes()))
2429         .setFamilyName(HBaseZeroCopyByteString.wrap(family))
2430         .setStoreHomeDir(storeDir.getName()); //make relative
2431     for (Path inputPath : inputPaths) {
2432       builder.addCompactionInput(inputPath.getName()); //relative path
2433     }
2434     for (Path outputPath : outputPaths) {
2435       builder.addCompactionOutput(outputPath.getName());
2436     }
2437     return builder.build();
2438   }
2439 
2440   /**
2441    * Return short version of Message toString'd, shorter than TextFormat#shortDebugString.
2442    * Tries to NOT print out data both because it can be big but also so we do not have data in our
2443    * logs. Use judiciously.
2444    * @param m
2445    * @return toString of passed <code>m</code>
2446    */
2447   public static String getShortTextFormat(Message m) {
2448     if (m == null) return "null";
2449     if (m instanceof ScanRequest) {
2450       // This should be small and safe to output.  No data.
2451       return TextFormat.shortDebugString(m);
2452     } else if (m instanceof RegionServerReportRequest) {
2453       // Print a short message only, just the servername and the requests, not the full load.
2454       RegionServerReportRequest r = (RegionServerReportRequest)m;
2455       return "server " + TextFormat.shortDebugString(r.getServer()) +
2456         " load { numberOfRequests: " + r.getLoad().getNumberOfRequests() + " }";
2457     } else if (m instanceof RegionServerStartupRequest) {
2458       // Should be small enough.
2459       return TextFormat.shortDebugString(m);
2460     } else if (m instanceof MutationProto) {
2461       return toShortString((MutationProto)m);
2462     } else if (m instanceof GetRequest) {
2463       GetRequest r = (GetRequest) m;
2464       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2465           ", row=" + getStringForByteString(r.getGet().getRow());
2466     } else if (m instanceof ClientProtos.MultiRequest) {
2467       ClientProtos.MultiRequest r = (ClientProtos.MultiRequest) m;
2468       // Get first set of Actions.
2469       ClientProtos.RegionAction actions = r.getRegionActionList().get(0);
2470       String row = actions.getActionCount() <= 0? "":
2471         getStringForByteString(actions.getAction(0).hasGet()?
2472           actions.getAction(0).getGet().getRow():
2473           actions.getAction(0).getMutation().getRow());
2474       return "region= " + getStringForByteString(actions.getRegion().getValue()) +
2475           ", for " + r.getRegionActionCount() +
2476           " actions and 1st row key=" + row;
2477     } else if (m instanceof ClientProtos.MutateRequest) {
2478       ClientProtos.MutateRequest r = (ClientProtos.MutateRequest) m;
2479       return "region= " + getStringForByteString(r.getRegion().getValue()) +
2480           ", row=" + getStringForByteString(r.getMutation().getRow());
2481     }
2482     return "TODO: " + m.getClass().toString();
2483   }
2484 
2485   private static String getStringForByteString(ByteString bs) {
2486     return Bytes.toStringBinary(bs.toByteArray());
2487   }
2488 
2489   /**
2490    * Print out some subset of a MutationProto rather than all of it and its data
2491    * @param proto Protobuf to print out
2492    * @return Short String of mutation proto
2493    */
2494   static String toShortString(final MutationProto proto) {
2495     return "row=" + Bytes.toString(proto.getRow().toByteArray()) +
2496         ", type=" + proto.getMutateType().toString();
2497   }
2498 
2499   public static TableName toTableName(HBaseProtos.TableName tableNamePB) {
2500     return TableName.valueOf(tableNamePB.getNamespace().asReadOnlyByteBuffer(),
2501         tableNamePB.getQualifier().asReadOnlyByteBuffer());
2502   }
2503 
2504   public static HBaseProtos.TableName toProtoTableName(TableName tableName) {
2505     return HBaseProtos.TableName.newBuilder()
2506         .setNamespace(HBaseZeroCopyByteString.wrap(tableName.getNamespace()))
2507         .setQualifier(HBaseZeroCopyByteString.wrap(tableName.getQualifier())).build();
2508   }
2509 
2510   public static TableName[] getTableNameArray(List<HBaseProtos.TableName> tableNamesList) {
2511     if (tableNamesList == null) {
2512       return new TableName[0];
2513     }
2514     TableName[] tableNames = new TableName[tableNamesList.size()];
2515     for (int i = 0; i < tableNamesList.size(); i++) {
2516       tableNames[i] = toTableName(tableNamesList.get(i));
2517     }
2518     return tableNames;
2519   }
2520 
2521   /**
2522    * Convert a protocol buffer CellVisibility to a client CellVisibility
2523    *
2524    * @param proto
2525    * @return the converted client CellVisibility
2526    */
2527   public static CellVisibility toCellVisibility(ClientProtos.CellVisibility proto) {
2528     if (proto == null) return null;
2529     return new CellVisibility(proto.getExpression());
2530   }
2531 
2532   /**
2533    * Convert a protocol buffer CellVisibility bytes to a client CellVisibility
2534    *
2535    * @param protoBytes
2536    * @return the converted client CellVisibility
2537    * @throws DeserializationException
2538    */
2539   public static CellVisibility toCellVisibility(byte[] protoBytes) throws DeserializationException {
2540     if (protoBytes == null) return null;
2541     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2542     ClientProtos.CellVisibility proto = null;
2543     try {
2544       proto = builder.mergeFrom(protoBytes).build();
2545     } catch (InvalidProtocolBufferException e) {
2546       throw new DeserializationException(e);
2547     }
2548     return toCellVisibility(proto);
2549   }
2550 
2551   /**
2552    * Create a protocol buffer CellVisibility based on a client CellVisibility.
2553    *
2554    * @param cellVisibility
2555    * @return a protocol buffer CellVisibility
2556    */
2557   public static ClientProtos.CellVisibility toCellVisibility(CellVisibility cellVisibility) {
2558     ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
2559     builder.setExpression(cellVisibility.getExpression());
2560     return builder.build();
2561   }
2562 
2563   /**
2564    * Convert a protocol buffer Authorizations to a client Authorizations
2565    *
2566    * @param proto
2567    * @return the converted client Authorizations
2568    */
2569   public static Authorizations toAuthorizations(ClientProtos.Authorizations proto) {
2570     if (proto == null) return null;
2571     return new Authorizations(proto.getLabelList());
2572   }
2573 
2574   /**
2575    * Convert a protocol buffer Authorizations bytes to a client Authorizations
2576    *
2577    * @param protoBytes
2578    * @return the converted client Authorizations
2579    * @throws DeserializationException
2580    */
2581   public static Authorizations toAuthorizations(byte[] protoBytes) throws DeserializationException {
2582     if (protoBytes == null) return null;
2583     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2584     ClientProtos.Authorizations proto = null;
2585     try {
2586       proto = builder.mergeFrom(protoBytes).build();
2587     } catch (InvalidProtocolBufferException e) {
2588       throw new DeserializationException(e);
2589     }
2590     return toAuthorizations(proto);
2591   }
2592 
2593   /**
2594    * Create a protocol buffer Authorizations based on a client Authorizations.
2595    *
2596    * @param authorizations
2597    * @return a protocol buffer Authorizations
2598    */
2599   public static ClientProtos.Authorizations toAuthorizations(Authorizations authorizations) {
2600     ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder();
2601     for (String label : authorizations.getLabels()) {
2602       builder.addLabel(label);
2603     }
2604     return builder.build();
2605   }
2606 
2607   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(String user,
2608       Permission perms) {
2609     return AccessControlProtos.UsersAndPermissions.newBuilder()
2610       .addUserPermissions(AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder()
2611         .setUser(ByteString.copyFromUtf8(user))
2612         .addPermissions(toPermission(perms))
2613         .build())
2614       .build();
2615   }
2616 
2617   public static AccessControlProtos.UsersAndPermissions toUsersAndPermissions(
2618       ListMultimap<String, Permission> perms) {
2619     AccessControlProtos.UsersAndPermissions.Builder builder =
2620         AccessControlProtos.UsersAndPermissions.newBuilder();
2621     for (Map.Entry<String, Collection<Permission>> entry : perms.asMap().entrySet()) {
2622       AccessControlProtos.UsersAndPermissions.UserPermissions.Builder userPermBuilder =
2623         AccessControlProtos.UsersAndPermissions.UserPermissions.newBuilder();
2624       userPermBuilder.setUser(ByteString.copyFromUtf8(entry.getKey()));
2625       for (Permission perm: entry.getValue()) {
2626         userPermBuilder.addPermissions(toPermission(perm));
2627       }
2628       builder.addUserPermissions(userPermBuilder.build());
2629     }
2630     return builder.build();
2631   }
2632 
2633   public static ListMultimap<String, Permission> toUsersAndPermissions(
2634       AccessControlProtos.UsersAndPermissions proto) {
2635     ListMultimap<String, Permission> result = ArrayListMultimap.create();
2636     for (AccessControlProtos.UsersAndPermissions.UserPermissions userPerms:
2637         proto.getUserPermissionsList()) {
2638       String user = userPerms.getUser().toStringUtf8();
2639       for (AccessControlProtos.Permission perm: userPerms.getPermissionsList()) {
2640         result.put(user, toPermission(perm));
2641       }
2642     }
2643     return result;
2644   }
2645 }