View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.protobuf;
19  
20  import java.io.IOException;
21  import java.util.ArrayList;
22  import java.util.HashMap;
23  import java.util.List;
24  import java.util.Map;
25
26  import edu.umd.cs.findbugs.annotations.Nullable;
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.hbase.Cell;
30  import org.apache.hadoop.hbase.CellScanner;
31  import org.apache.hadoop.hbase.DoNotRetryIOException;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.ServerName;
34  import org.apache.hadoop.hbase.classification.InterfaceAudience;
35  import org.apache.hadoop.hbase.client.Result;
36  import org.apache.hadoop.hbase.client.SingleResponse;
37  import org.apache.hadoop.hbase.ipc.ServerRpcController;
38  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.GetUserPermissionsResponse;
39  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
40  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
41  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
42  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
43  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
44  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
45  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
46  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
47  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
48  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
49  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
50  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
51  import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
52  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
53  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
54  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameInt64Pair;
55  import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos.ScanMetrics;
56  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorResponse;
57  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanResponse;
58  import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
59  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
60  import org.apache.hadoop.hbase.security.access.UserPermission;
61  import org.apache.hadoop.util.StringUtils;
62
63  import com.google.protobuf.ByteString;
64  import com.google.protobuf.RpcController;
65
66  /**
67   * Helper utility to build protocol buffer responses,
68   * or retrieve data from protocol buffer responses.
69   */
70  @InterfaceAudience.Private
71  public final class ResponseConverter {
72    private static final Log LOG = LogFactory.getLog(ResponseConverter.class);
73
74    private ResponseConverter() {
75    }
76
77  // Start utilities for Client
78
79    /**
80     * Get the results from a protocol buffer MultiResponse
81     *
82     * @param request the protocol buffer MultiResponse to convert
83     * @param cells Cells to go with the passed in <code>proto</code>.  Can be null.
84     * @return the results that were in the MultiResponse (a Result or an Exception).
85     * @throws IOException
86     */
87    public static org.apache.hadoop.hbase.client.MultiResponse getResults(final MultiRequest request,
88        final MultiResponse response, final CellScanner cells)
89    throws IOException {
90      int requestRegionActionCount = request.getRegionActionCount();
91      int responseRegionActionResultCount = response.getRegionActionResultCount();
92      if (requestRegionActionCount != responseRegionActionResultCount) {
93        throw new IllegalStateException("Request mutation count=" + requestRegionActionCount +
94            " does not match response mutation result count=" + responseRegionActionResultCount);
95      }
96
97      org.apache.hadoop.hbase.client.MultiResponse results =
98        new org.apache.hadoop.hbase.client.MultiResponse();
99
100     for (int i = 0; i < responseRegionActionResultCount; i++) {
101       RegionAction actions = request.getRegionAction(i);
102       RegionActionResult actionResult = response.getRegionActionResult(i);
103       HBaseProtos.RegionSpecifier rs = actions.getRegion();
104       if (rs.hasType() &&
105           (rs.getType() != HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME)){
106         throw new IllegalArgumentException(
107             "We support only encoded types for protobuf multi response.");
108       }
109       byte[] regionName = rs.getValue().toByteArray();
110
111       if (actionResult.hasException()) {
112         Throwable regionException =  ProtobufUtil.toException(actionResult.getException());
113         results.addException(regionName, regionException);
114         continue;
115       }
116
117       if (actions.getActionCount() != actionResult.getResultOrExceptionCount()) {
118         throw new IllegalStateException("actions.getActionCount=" + actions.getActionCount() +
119             ", actionResult.getResultOrExceptionCount=" +
120             actionResult.getResultOrExceptionCount() + " for region " + actions.getRegion());
121       }
122
123       for (ResultOrException roe : actionResult.getResultOrExceptionList()) {
124         Object responseValue;
125         if (roe.hasException()) {
126           responseValue = ProtobufUtil.toException(roe.getException());
127         } else if (roe.hasResult()) {
128           responseValue = ProtobufUtil.toResult(roe.getResult(), cells);
129         } else if (roe.hasServiceResult()) {
130           responseValue = roe.getServiceResult();
131         } else{
132           // Sometimes, the response is just "it was processed". Generally, this occurs for things
133           // like mutateRows where either we get back 'processed' (or not) and optionally some
134           // statistics about the regions we touched.
135           responseValue = response.getProcessed() ?
136                           ProtobufUtil.EMPTY_RESULT_EXISTS_TRUE :
137                           ProtobufUtil.EMPTY_RESULT_EXISTS_FALSE;
138         }
139         results.add(regionName, roe.getIndex(), responseValue);
140       }
141     }
142
143     if (response.hasRegionStatistics()) {
144       ClientProtos.MultiRegionLoadStats stats = response.getRegionStatistics();
145       for (int i = 0; i < stats.getRegionCount(); i++) {
146         results.addStatistic(stats.getRegion(i).getValue().toByteArray(), stats.getStat(i));
147       }
148     }
149
150     return results;
151   }
152
153
154   public static SingleResponse getResult(final ClientProtos.MutateRequest request,
155                                          final ClientProtos.MutateResponse response,
156                                          final CellScanner cells)
157       throws IOException {
158     SingleResponse singleResponse = new SingleResponse();
159     SingleResponse.Entry entry = new SingleResponse.Entry();
160     entry.setResult(ProtobufUtil.toResult(response.getResult(), cells));
161     entry.setProcessed(response.getProcessed());
162     singleResponse.setEntry(entry);
163     return singleResponse;
164   }
165
166   /**
167    * Wrap a throwable to an action result.
168    *
169    * @param t
170    * @return an action result builder
171    */
172   public static ResultOrException.Builder buildActionResult(final Throwable t) {
173     ResultOrException.Builder builder = ResultOrException.newBuilder();
174     if (t != null) builder.setException(buildException(t));
175     return builder;
176   }
177
178   /**
179    * Wrap a throwable to an action result.
180    *
181    * @param r
182    * @return an action result builder
183    */
184   public static ResultOrException.Builder buildActionResult(final ClientProtos.Result r) {
185     ResultOrException.Builder builder = ResultOrException.newBuilder();
186     if (r != null) builder.setResult(r);
187     return builder;
188   }
189
190   /**
191    * @param t
192    * @return NameValuePair of the exception name to stringified version os exception.
193    */
194   public static NameBytesPair buildException(final Throwable t) {
195     NameBytesPair.Builder parameterBuilder = NameBytesPair.newBuilder();
196     parameterBuilder.setName(t.getClass().getName());
197     parameterBuilder.setValue(
198       ByteString.copyFromUtf8(StringUtils.stringifyException(t)));
199     return parameterBuilder.build();
200   }
201
202   /**
203    * Converts the permissions list into a protocol buffer GetUserPermissionsResponse
204    */
205   public static GetUserPermissionsResponse buildGetUserPermissionsResponse(
206       final List<UserPermission> permissions) {
207     GetUserPermissionsResponse.Builder builder = GetUserPermissionsResponse.newBuilder();
208     for (UserPermission perm : permissions) {
209       builder.addUserPermission(ProtobufUtil.toUserPermission(perm));
210     }
211     return builder.build();
212   }
213 
214 // End utilities for Client
215 // Start utilities for Admin
216
217   /**
218    * Get the list of region info from a GetOnlineRegionResponse
219    *
220    * @param proto the GetOnlineRegionResponse
221    * @return the list of region info
222    */
223   public static List<HRegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
224     if (proto == null || proto.getRegionInfoCount() == 0) return null;
225     return ProtobufUtil.getRegionInfos(proto);
226   }
227
228   /**
229    * Get the region opening state from a OpenRegionResponse
230    *
231    * @param proto the OpenRegionResponse
232    * @return the region opening state
233    */
234   public static RegionOpeningState getRegionOpeningState
235       (final OpenRegionResponse proto) {
236     if (proto == null || proto.getOpeningStateCount() != 1) return null;
237     return RegionOpeningState.valueOf(
238       proto.getOpeningState(0).name());
239   }
240
241   /**
242    * Get a list of region opening state from a OpenRegionResponse
243    * 
244    * @param proto the OpenRegionResponse
245    * @return the list of region opening state
246    */
247   public static List<RegionOpeningState> getRegionOpeningStateList(
248       final OpenRegionResponse proto) {
249     if (proto == null) return null;
250     List<RegionOpeningState> regionOpeningStates = new ArrayList<RegionOpeningState>();
251     for (int i = 0; i < proto.getOpeningStateCount(); i++) {
252       regionOpeningStates.add(RegionOpeningState.valueOf(
253           proto.getOpeningState(i).name()));
254     }
255     return regionOpeningStates;
256   }
257
258   /**
259    * Check if the region is closed from a CloseRegionResponse
260    *
261    * @param proto the CloseRegionResponse
262    * @return the region close state
263    */
264   public static boolean isClosed
265       (final CloseRegionResponse proto) {
266     if (proto == null || !proto.hasClosed()) return false;
267     return proto.getClosed();
268   }
269
270   /**
271    * A utility to build a GetServerInfoResponse.
272    *
273    * @param serverName
274    * @param webuiPort
275    * @return the response
276    */
277   public static GetServerInfoResponse buildGetServerInfoResponse(
278       final ServerName serverName, final int webuiPort) {
279     GetServerInfoResponse.Builder builder = GetServerInfoResponse.newBuilder();
280     ServerInfo.Builder serverInfoBuilder = ServerInfo.newBuilder();
281     serverInfoBuilder.setServerName(ProtobufUtil.toServerName(serverName));
282     if (webuiPort >= 0) {
283       serverInfoBuilder.setWebuiPort(webuiPort);
284     }
285     builder.setServerInfo(serverInfoBuilder.build());
286     return builder.build();
287   }
288
289   /**
290    * A utility to build a GetOnlineRegionResponse.
291    *
292    * @param regions
293    * @return the response
294    */
295   public static GetOnlineRegionResponse buildGetOnlineRegionResponse(
296       final List<HRegionInfo> regions) {
297     GetOnlineRegionResponse.Builder builder = GetOnlineRegionResponse.newBuilder();
298     for (HRegionInfo region: regions) {
299       builder.addRegionInfo(HRegionInfo.convert(region));
300     }
301     return builder.build();
302   }
303
304   /**
305    * Creates a response for the catalog scan request
306    * @return A RunCatalogScanResponse
307    */
308   public static RunCatalogScanResponse buildRunCatalogScanResponse(int numCleaned) {
309     return RunCatalogScanResponse.newBuilder().setScanResult(numCleaned).build();
310   }
311
312   /**
313    * Creates a response for the catalog scan request
314    * @return A EnableCatalogJanitorResponse
315    */
316   public static EnableCatalogJanitorResponse buildEnableCatalogJanitorResponse(boolean prevValue) {
317     return EnableCatalogJanitorResponse.newBuilder().setPrevValue(prevValue).build();
318   }
319
320 // End utilities for Admin
321
322   /**
323    * Creates a response for the last flushed sequence Id request
324    * @return A GetLastFlushedSequenceIdResponse
325    */
326   public static GetLastFlushedSequenceIdResponse buildGetLastFlushedSequenceIdResponse(
327       RegionStoreSequenceIds ids) {
328     return GetLastFlushedSequenceIdResponse.newBuilder()
329         .setLastFlushedSequenceId(ids.getLastFlushedSequenceId())
330         .addAllStoreLastFlushedSequenceId(ids.getStoreSequenceIdList()).build();
331   }
332
333   /**
334    * Stores an exception encountered during RPC invocation so it can be passed back
335    * through to the client.
336    * @param controller the controller instance provided by the client when calling the service
337    * @param ioe the exception encountered
338    */
339   public static void setControllerException(RpcController controller, IOException ioe) {
340     if (controller != null) {
341       if (controller instanceof ServerRpcController) {
342         ((ServerRpcController)controller).setFailedOn(ioe);
343       } else {
344         controller.setFailed(StringUtils.stringifyException(ioe));
345       }
346     }
347   }
348
349   /**
350    * Retreivies exception stored during RPC invocation.
351    * @param controller the controller instance provided by the client when calling the service
352    * @return exception if any, or null; Will return DoNotRetryIOException for string represented
353    * failure causes in controller.
354    */
355   @Nullable
356   public static IOException getControllerException(RpcController controller) throws IOException {
357     if (controller != null && controller.failed()) {
358       if (controller instanceof ServerRpcController) {
359         return ((ServerRpcController)controller).getFailedOn();
360       } else {
361         return new DoNotRetryIOException(controller.errorText());
362       }
363     }
364     return null;
365   }
366
367
368   /**
369    * Create Results from the cells using the cells meta data. 
370    * @param cellScanner
371    * @param response
372    * @return results
373    */
374   public static Result[] getResults(CellScanner cellScanner, ScanResponse response)
375       throws IOException {
376     if (response == null) return null;
377     // If cellscanner, then the number of Results to return is the count of elements in the
378     // cellsPerResult list.  Otherwise, it is how many results are embedded inside the response.
379     int noOfResults = cellScanner != null?
380       response.getCellsPerResultCount(): response.getResultsCount();
381     Result[] results = new Result[noOfResults];
382     for (int i = 0; i < noOfResults; i++) {
383       if (cellScanner != null) {
384         // Cells are out in cellblocks.  Group them up again as Results.  How many to read at a
385         // time will be found in getCellsLength -- length here is how many Cells in the i'th Result
386         int noOfCells = response.getCellsPerResult(i);
387         boolean isPartial =
388             response.getPartialFlagPerResultCount() > i ?
389                 response.getPartialFlagPerResult(i) : false;
390         List<Cell> cells = new ArrayList<Cell>(noOfCells);
391         for (int j = 0; j < noOfCells; j++) {
392           try {
393             if (cellScanner.advance() == false) {
394               // We are not able to retrieve the exact number of cells which ResultCellMeta says us.
395               // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
396               // same scanner will result in OutOfOrderScannerNextException
397               String msg = "Results sent from server=" + noOfResults + ". But only got " + i
398                 + " results completely at client. Resetting the scanner to scan again.";
399               LOG.error(msg);
400               throw new DoNotRetryIOException(msg);
401             }
402           } catch (IOException ioe) {
403             // We are getting IOE while retrieving the cells for Results.
404             // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
405             // same scanner will result in OutOfOrderScannerNextException
406             LOG.error("Exception while reading cells from result."
407               + "Resetting the scanner to scan again.", ioe);
408             throw new DoNotRetryIOException("Resetting the scanner.", ioe);
409           }
410           cells.add(cellScanner.current());
411         }
412         results[i] = Result.create(cells, null, response.getStale(), isPartial);
413       } else {
414         // Result is pure pb.
415         results[i] = ProtobufUtil.toResult(response.getResults(i));
416       }
417     }
418     return results;
419   }
420
421   public static Map<String, Long> getScanMetrics(ScanResponse response) {
422     Map<String, Long> metricMap = new HashMap<String, Long>();
423     if (response == null || !response.hasScanMetrics() || response.getScanMetrics() == null) {
424       return metricMap;
425     }
426
427     ScanMetrics metrics = response.getScanMetrics();
428     int numberOfMetrics = metrics.getMetricsCount();
429     for (int i = 0; i < numberOfMetrics; i++) {
430       NameInt64Pair metricPair = metrics.getMetrics(i);
431       if (metricPair != null) {
432         String name = metricPair.getName();
433         Long value = metricPair.getValue();
434         if (name != null && value != null) {
435           metricMap.put(name, value);
436         }
437       }
438     }
439
440     return metricMap;
441   }
442 }