View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.protobuf;
19  
20  import javax.annotation.Nullable;
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.HashMap;
24  import java.util.List;
25  import java.util.Map;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.hbase.Cell;
30  import org.apache.hadoop.hbase.CellScanner;
31  import org.apache.hadoop.hbase.DoNotRetryIOException;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.ServerName;
34  import org.apache.hadoop.hbase.classification.InterfaceAudience;
35  import org.apache.hadoop.hbase.client.Result;
36  import org.apache.hadoop.hbase.ipc.ServerRpcController;
37  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.GetUserPermissionsResponse;
38  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
39  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
40  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
41  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
42  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
43  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
44  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
45  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
46  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
47  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
48  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
49  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
50  import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
51  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
52  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
53  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair;
54  import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics;
55  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorResponse;
56  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanResponse;
57  import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
58  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
59  import org.apache.hadoop.hbase.security.access.UserPermission;
60  import org.apache.hadoop.util.StringUtils;
61  
62  import com.google.protobuf.ByteString;
63  import com.google.protobuf.RpcController;
64  
65  /**
66   * Helper utility to build protocol buffer responses,
67   * or retrieve data from protocol buffer responses.
68   */
69  @InterfaceAudience.Private
70  public final class ResponseConverter {
71    private static final Log LOG = LogFactory.getLog(ResponseConverter.class);
72  
73    private ResponseConverter() {
74    }
75  
76  // Start utilities for Client
77  
78    /**
79     * Get the results from a protocol buffer MultiResponse
80     *
81     * @param request the protocol buffer MultiResponse to convert
82     * @param cells Cells to go with the passed in <code>proto</code>.  Can be null.
83     * @return the results that were in the MultiResponse (a Result or an Exception).
84     * @throws IOException
85     */
86    public static org.apache.hadoop.hbase.client.MultiResponse getResults(final MultiRequest request,
87        final MultiResponse response, final CellScanner cells)
88    throws IOException {
89      int requestRegionActionCount = request.getRegionActionCount();
90      int responseRegionActionResultCount = response.getRegionActionResultCount();
91      if (requestRegionActionCount != responseRegionActionResultCount) {
92        throw new IllegalStateException("Request mutation count=" + requestRegionActionCount +
93            " does not match response mutation result count=" + responseRegionActionResultCount);
94      }
95  
96      org.apache.hadoop.hbase.client.MultiResponse results =
97        new org.apache.hadoop.hbase.client.MultiResponse();
98  
99      for (int i = 0; i < responseRegionActionResultCount; i++) {
100       RegionAction actions = request.getRegionAction(i);
101       RegionActionResult actionResult = response.getRegionActionResult(i);
102       HBaseProtos.RegionSpecifier rs = actions.getRegion();
103       if (rs.hasType() &&
104           (rs.getType() != HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME)){
105         throw new IllegalArgumentException(
106             "We support only encoded types for protobuf multi response.");
107       }
108       byte[] regionName = rs.getValue().toByteArray();
109 
110       if (actionResult.hasException()) {
111         Throwable regionException =  ProtobufUtil.toException(actionResult.getException());
112         results.addException(regionName, regionException);
113         continue;
114       }
115 
116       if (actions.getActionCount() != actionResult.getResultOrExceptionCount()) {
117         throw new IllegalStateException("actions.getActionCount=" + actions.getActionCount() +
118             ", actionResult.getResultOrExceptionCount=" +
119             actionResult.getResultOrExceptionCount() + " for region " + actions.getRegion());
120       }
121 
122       for (ResultOrException roe : actionResult.getResultOrExceptionList()) {
123         Object responseValue;
124         if (roe.hasException()) {
125           responseValue = ProtobufUtil.toException(roe.getException());
126         } else if (roe.hasResult()) {
127           responseValue = ProtobufUtil.toResult(roe.getResult(), cells);
128         } else if (roe.hasServiceResult()) {
129           responseValue = roe.getServiceResult();
130         } else{
131           // Sometimes, the response is just "it was processed". Generally, this occurs for things
132           // like mutateRows where either we get back 'processed' (or not) and optionally some
133           // statistics about the regions we touched.
134           responseValue = response.getProcessed() ?
135                           ProtobufUtil.EMPTY_RESULT_EXISTS_TRUE :
136                           ProtobufUtil.EMPTY_RESULT_EXISTS_FALSE;
137         }
138         results.add(regionName, roe.getIndex(), responseValue);
139       }
140     }
141 
142     if (response.hasRegionStatistics()) {
143       ClientProtos.MultiRegionLoadStats stats = response.getRegionStatistics();
144       for (int i = 0; i < stats.getRegionCount(); i++) {
145         results.addStatistic(stats.getRegion(i).getValue().toByteArray(), stats.getStat(i));
146       }
147     }
148 
149     return results;
150   }
151 
152   /**
153    * Wrap a throwable to an action result.
154    *
155    * @param t
156    * @return an action result builder
157    */
158   public static ResultOrException.Builder buildActionResult(final Throwable t) {
159     ResultOrException.Builder builder = ResultOrException.newBuilder();
160     if (t != null) builder.setException(buildException(t));
161     return builder;
162   }
163 
164   /**
165    * Wrap a throwable to an action result.
166    *
167    * @param r
168    * @return an action result builder
169    */
170   public static ResultOrException.Builder buildActionResult(final ClientProtos.Result r) {
171     ResultOrException.Builder builder = ResultOrException.newBuilder();
172     if (r != null) builder.setResult(r);
173     return builder;
174   }
175 
176   /**
177    * @param t
178    * @return NameValuePair of the exception name to stringified version os exception.
179    */
180   public static NameBytesPair buildException(final Throwable t) {
181     NameBytesPair.Builder parameterBuilder = NameBytesPair.newBuilder();
182     parameterBuilder.setName(t.getClass().getName());
183     parameterBuilder.setValue(
184       ByteString.copyFromUtf8(StringUtils.stringifyException(t)));
185     return parameterBuilder.build();
186   }
187 
188   /**
189    * Converts the permissions list into a protocol buffer GetUserPermissionsResponse
190    */
191   public static GetUserPermissionsResponse buildGetUserPermissionsResponse(
192       final List<UserPermission> permissions) {
193     GetUserPermissionsResponse.Builder builder = GetUserPermissionsResponse.newBuilder();
194     for (UserPermission perm : permissions) {
195       builder.addUserPermission(ProtobufUtil.toUserPermission(perm));
196     }
197     return builder.build();
198   }
199 
200 // End utilities for Client
201 // Start utilities for Admin
202 
203   /**
204    * Get the list of region info from a GetOnlineRegionResponse
205    *
206    * @param proto the GetOnlineRegionResponse
207    * @return the list of region info
208    */
209   public static List<HRegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
210     if (proto == null || proto.getRegionInfoCount() == 0) return null;
211     return ProtobufUtil.getRegionInfos(proto);
212   }
213 
214   /**
215    * Get the region opening state from a OpenRegionResponse
216    *
217    * @param proto the OpenRegionResponse
218    * @return the region opening state
219    */
220   public static RegionOpeningState getRegionOpeningState
221       (final OpenRegionResponse proto) {
222     if (proto == null || proto.getOpeningStateCount() != 1) return null;
223     return RegionOpeningState.valueOf(
224       proto.getOpeningState(0).name());
225   }
226 
227   /**
228    * Get a list of region opening state from a OpenRegionResponse
229    * 
230    * @param proto the OpenRegionResponse
231    * @return the list of region opening state
232    */
233   public static List<RegionOpeningState> getRegionOpeningStateList(
234       final OpenRegionResponse proto) {
235     if (proto == null) return null;
236     List<RegionOpeningState> regionOpeningStates = new ArrayList<RegionOpeningState>();
237     for (int i = 0; i < proto.getOpeningStateCount(); i++) {
238       regionOpeningStates.add(RegionOpeningState.valueOf(
239           proto.getOpeningState(i).name()));
240     }
241     return regionOpeningStates;
242   }
243 
244   /**
245    * Check if the region is closed from a CloseRegionResponse
246    *
247    * @param proto the CloseRegionResponse
248    * @return the region close state
249    */
250   public static boolean isClosed
251       (final CloseRegionResponse proto) {
252     if (proto == null || !proto.hasClosed()) return false;
253     return proto.getClosed();
254   }
255 
256   /**
257    * A utility to build a GetServerInfoResponse.
258    *
259    * @param serverName
260    * @param webuiPort
261    * @return the response
262    */
263   public static GetServerInfoResponse buildGetServerInfoResponse(
264       final ServerName serverName, final int webuiPort) {
265     GetServerInfoResponse.Builder builder = GetServerInfoResponse.newBuilder();
266     ServerInfo.Builder serverInfoBuilder = ServerInfo.newBuilder();
267     serverInfoBuilder.setServerName(ProtobufUtil.toServerName(serverName));
268     if (webuiPort >= 0) {
269       serverInfoBuilder.setWebuiPort(webuiPort);
270     }
271     builder.setServerInfo(serverInfoBuilder.build());
272     return builder.build();
273   }
274 
275   /**
276    * A utility to build a GetOnlineRegionResponse.
277    *
278    * @param regions
279    * @return the response
280    */
281   public static GetOnlineRegionResponse buildGetOnlineRegionResponse(
282       final List<HRegionInfo> regions) {
283     GetOnlineRegionResponse.Builder builder = GetOnlineRegionResponse.newBuilder();
284     for (HRegionInfo region: regions) {
285       builder.addRegionInfo(HRegionInfo.convert(region));
286     }
287     return builder.build();
288   }
289 
290   /**
291    * Creates a response for the catalog scan request
292    * @return A RunCatalogScanResponse
293    */
294   public static RunCatalogScanResponse buildRunCatalogScanResponse(int numCleaned) {
295     return RunCatalogScanResponse.newBuilder().setScanResult(numCleaned).build();
296   }
297 
298   /**
299    * Creates a response for the catalog scan request
300    * @return A EnableCatalogJanitorResponse
301    */
302   public static EnableCatalogJanitorResponse buildEnableCatalogJanitorResponse(boolean prevValue) {
303     return EnableCatalogJanitorResponse.newBuilder().setPrevValue(prevValue).build();
304   }
305 
306 // End utilities for Admin
307 
308   /**
309    * Creates a response for the last flushed sequence Id request
310    * @return A GetLastFlushedSequenceIdResponse
311    */
312   public static GetLastFlushedSequenceIdResponse buildGetLastFlushedSequenceIdResponse(
313       RegionStoreSequenceIds ids) {
314     return GetLastFlushedSequenceIdResponse.newBuilder()
315         .setLastFlushedSequenceId(ids.getLastFlushedSequenceId())
316         .addAllStoreLastFlushedSequenceId(ids.getStoreSequenceIdList()).build();
317   }
318 
319   /**
320    * Stores an exception encountered during RPC invocation so it can be passed back
321    * through to the client.
322    * @param controller the controller instance provided by the client when calling the service
323    * @param ioe the exception encountered
324    */
325   public static void setControllerException(RpcController controller, IOException ioe) {
326     if (controller != null) {
327       if (controller instanceof ServerRpcController) {
328         ((ServerRpcController)controller).setFailedOn(ioe);
329       } else {
330         controller.setFailed(StringUtils.stringifyException(ioe));
331       }
332     }
333   }
334 
335   /**
336    * Retreivies exception stored during RPC invocation.
337    * @param controller the controller instance provided by the client when calling the service
338    * @return exception if any, or null; Will return DoNotRetryIOException for string represented
339    * failure causes in controller.
340    */
341   @Nullable
342   public static IOException getControllerException(RpcController controller) throws IOException {
343     if (controller != null && controller.failed()) {
344       if (controller instanceof ServerRpcController) {
345         return ((ServerRpcController)controller).getFailedOn();
346       } else {
347         return new DoNotRetryIOException(controller.errorText());
348       }
349     }
350     return null;
351   }
352 
353 
354   /**
355    * Create Results from the cells using the cells meta data. 
356    * @param cellScanner
357    * @param response
358    * @return results
359    */
360   public static Result[] getResults(CellScanner cellScanner, ScanResponse response)
361       throws IOException {
362     if (response == null) return null;
363     // If cellscanner, then the number of Results to return is the count of elements in the
364     // cellsPerResult list.  Otherwise, it is how many results are embedded inside the response.
365     int noOfResults = cellScanner != null?
366       response.getCellsPerResultCount(): response.getResultsCount();
367     Result[] results = new Result[noOfResults];
368     for (int i = 0; i < noOfResults; i++) {
369       if (cellScanner != null) {
370         // Cells are out in cellblocks.  Group them up again as Results.  How many to read at a
371         // time will be found in getCellsLength -- length here is how many Cells in the i'th Result
372         int noOfCells = response.getCellsPerResult(i);
373         boolean isPartial =
374             response.getPartialFlagPerResultCount() > i ?
375                 response.getPartialFlagPerResult(i) : false;
376         List<Cell> cells = new ArrayList<Cell>(noOfCells);
377         for (int j = 0; j < noOfCells; j++) {
378           try {
379             if (cellScanner.advance() == false) {
380               // We are not able to retrieve the exact number of cells which ResultCellMeta says us.
381               // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
382               // same scanner will result in OutOfOrderScannerNextException
383               String msg = "Results sent from server=" + noOfResults + ". But only got " + i
384                 + " results completely at client. Resetting the scanner to scan again.";
385               LOG.error(msg);
386               throw new DoNotRetryIOException(msg);
387             }
388           } catch (IOException ioe) {
389             // We are getting IOE while retrieving the cells for Results.
390             // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
391             // same scanner will result in OutOfOrderScannerNextException
392             LOG.error("Exception while reading cells from result."
393               + "Resetting the scanner to scan again.", ioe);
394             throw new DoNotRetryIOException("Resetting the scanner.", ioe);
395           }
396           cells.add(cellScanner.current());
397         }
398         results[i] = Result.create(cells, null, response.getStale(), isPartial);
399       } else {
400         // Result is pure pb.
401         results[i] = ProtobufUtil.toResult(response.getResults(i));
402       }
403     }
404     return results;
405   }
406 
407   public static Map<String, Long> getScanMetrics(ScanResponse response) {
408     Map<String, Long> metricMap = new HashMap<String, Long>();
409     if (response == null || !response.hasScanMetrics() || response.getScanMetrics() == null) {
410       return metricMap;
411     }
412     
413     ScanMetrics metrics = response.getScanMetrics();
414     int numberOfMetrics = metrics.getMetricsCount();
415     for (int i = 0; i < numberOfMetrics; i++) {
416       NameInt64Pair metricPair = metrics.getMetrics(i);
417       if (metricPair != null) {
418         String name = metricPair.getName();
419         Long value = metricPair.getValue();
420         if (name != null && value != null) {
421           metricMap.put(name, value);
422         }
423       }
424     }
425 
426     return metricMap;
427   }
428 }