View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.protobuf;
19  
20  import java.io.IOException;
21  import java.util.ArrayList;
22  import java.util.List;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.hbase.Cell;
27  import org.apache.hadoop.hbase.CellScanner;
28  import org.apache.hadoop.hbase.DoNotRetryIOException;
29  import org.apache.hadoop.hbase.HRegionInfo;
30  import org.apache.hadoop.hbase.ServerName;
31  import org.apache.hadoop.hbase.classification.InterfaceAudience;
32  import org.apache.hadoop.hbase.client.Result;
33  import org.apache.hadoop.hbase.ipc.ServerRpcController;
34  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.GetUserPermissionsResponse;
35  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
36  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
37  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
38  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
39  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
40  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
41  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
42  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
43  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
44  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
45  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
46  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
47  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
48  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
49  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorResponse;
50  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanResponse;
51  import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
52  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
53  import org.apache.hadoop.hbase.security.access.UserPermission;
54  import org.apache.hadoop.util.StringUtils;
55  
56  import com.google.protobuf.ByteString;
57  import com.google.protobuf.RpcController;
58  
59  /**
60   * Helper utility to build protocol buffer responses,
61   * or retrieve data from protocol buffer responses.
62   */
63  @InterfaceAudience.Private
64  public final class ResponseConverter {
65    public static final Log LOG = LogFactory.getLog(ResponseConverter.class);
66  
67    private ResponseConverter() {
68    }
69  
70  // Start utilities for Client
71  
72    /**
73     * Get the results from a protocol buffer MultiResponse
74     *
75     * @param request the protocol buffer MultiResponse to convert
76     * @param cells Cells to go with the passed in <code>proto</code>.  Can be null.
77     * @return the results that were in the MultiResponse (a Result or an Exception).
78     * @throws IOException
79     */
80    public static org.apache.hadoop.hbase.client.MultiResponse getResults(final MultiRequest request,
81        final MultiResponse response, final CellScanner cells)
82    throws IOException {
83      int requestRegionActionCount = request.getRegionActionCount();
84      int responseRegionActionResultCount = response.getRegionActionResultCount();
85      if (requestRegionActionCount != responseRegionActionResultCount) {
86        throw new IllegalStateException("Request mutation count=" + responseRegionActionResultCount +
87            " does not match response mutation result count=" + responseRegionActionResultCount);
88      }
89  
90      org.apache.hadoop.hbase.client.MultiResponse results =
91        new org.apache.hadoop.hbase.client.MultiResponse();
92  
93      for (int i = 0; i < responseRegionActionResultCount; i++) {
94        RegionAction actions = request.getRegionAction(i);
95        RegionActionResult actionResult = response.getRegionActionResult(i);
96        HBaseProtos.RegionSpecifier rs = actions.getRegion();
97        if (rs.hasType() &&
98            (rs.getType() != HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME)){
99          throw new IllegalArgumentException(
100             "We support only encoded types for protobuf multi response.");
101       }
102       byte[] regionName = rs.getValue().toByteArray();
103 
104       if (actionResult.hasException()) {
105         Throwable regionException =  ProtobufUtil.toException(actionResult.getException());
106         results.addException(regionName, regionException);
107         continue;
108       }
109 
110       if (actions.getActionCount() != actionResult.getResultOrExceptionCount()) {
111         throw new IllegalStateException("actions.getActionCount=" + actions.getActionCount() +
112             ", actionResult.getResultOrExceptionCount=" +
113             actionResult.getResultOrExceptionCount() + " for region " + actions.getRegion());
114       }
115 
116       for (ResultOrException roe : actionResult.getResultOrExceptionList()) {
117         Object responseValue;
118         if (roe.hasException()) {
119           responseValue = ProtobufUtil.toException(roe.getException());
120         } else if (roe.hasResult()) {
121           responseValue = ProtobufUtil.toResult(roe.getResult(), cells);
122           // add the load stats, if we got any
123           if (roe.hasLoadStats()) {
124             ((Result) responseValue).addResults(roe.getLoadStats());
125           }
126         } else if (roe.hasServiceResult()) {
127           responseValue = roe.getServiceResult();
128         } else {
129           // no result & no exception. Unexpected.
130           throw new IllegalStateException("No result & no exception roe=" + roe +
131               " for region " + actions.getRegion());
132         }
133         results.add(regionName, roe.getIndex(), responseValue);
134       }
135     }
136 
137     return results;
138   }
139 
140   /**
141    * Wrap a throwable to an action result.
142    *
143    * @param t
144    * @return an action result builder
145    */
146   public static ResultOrException.Builder buildActionResult(final Throwable t) {
147     ResultOrException.Builder builder = ResultOrException.newBuilder();
148     if (t != null) builder.setException(buildException(t));
149     return builder;
150   }
151 
152   /**
153    * Wrap a throwable to an action result.
154    *
155    * @param r
156    * @return an action result builder
157    */
158   public static ResultOrException.Builder buildActionResult(final ClientProtos.Result r,
159       ClientProtos.RegionLoadStats stats) {
160     ResultOrException.Builder builder = ResultOrException.newBuilder();
161     if (r != null) builder.setResult(r);
162     if(stats != null) builder.setLoadStats(stats);
163     return builder;
164   }
165 
166   /**
167    * @param t
168    * @return NameValuePair of the exception name to stringified version os exception.
169    */
170   public static NameBytesPair buildException(final Throwable t) {
171     NameBytesPair.Builder parameterBuilder = NameBytesPair.newBuilder();
172     parameterBuilder.setName(t.getClass().getName());
173     parameterBuilder.setValue(
174       ByteString.copyFromUtf8(StringUtils.stringifyException(t)));
175     return parameterBuilder.build();
176   }
177 
178   /**
179    * Converts the permissions list into a protocol buffer GetUserPermissionsResponse
180    */
181   public static GetUserPermissionsResponse buildGetUserPermissionsResponse(
182       final List<UserPermission> permissions) {
183     GetUserPermissionsResponse.Builder builder = GetUserPermissionsResponse.newBuilder();
184     for (UserPermission perm : permissions) {
185       builder.addUserPermission(ProtobufUtil.toUserPermission(perm));
186     }
187     return builder.build();
188   }
189 
190 // End utilities for Client
191 // Start utilities for Admin
192 
193   /**
194    * Get the list of region info from a GetOnlineRegionResponse
195    *
196    * @param proto the GetOnlineRegionResponse
197    * @return the list of region info
198    */
199   public static List<HRegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
200     if (proto == null || proto.getRegionInfoCount() == 0) return null;
201     return ProtobufUtil.getRegionInfos(proto);
202   }
203 
204   /**
205    * Get the region opening state from a OpenRegionResponse
206    *
207    * @param proto the OpenRegionResponse
208    * @return the region opening state
209    */
210   public static RegionOpeningState getRegionOpeningState
211       (final OpenRegionResponse proto) {
212     if (proto == null || proto.getOpeningStateCount() != 1) return null;
213     return RegionOpeningState.valueOf(
214       proto.getOpeningState(0).name());
215   }
216 
217   /**
218    * Get a list of region opening state from a OpenRegionResponse
219    * 
220    * @param proto the OpenRegionResponse
221    * @return the list of region opening state
222    */
223   public static List<RegionOpeningState> getRegionOpeningStateList(
224       final OpenRegionResponse proto) {
225     if (proto == null) return null;
226     List<RegionOpeningState> regionOpeningStates = new ArrayList<RegionOpeningState>();
227     for (int i = 0; i < proto.getOpeningStateCount(); i++) {
228       regionOpeningStates.add(RegionOpeningState.valueOf(
229           proto.getOpeningState(i).name()));
230     }
231     return regionOpeningStates;
232   }
233 
234   /**
235    * Check if the region is closed from a CloseRegionResponse
236    *
237    * @param proto the CloseRegionResponse
238    * @return the region close state
239    */
240   public static boolean isClosed
241       (final CloseRegionResponse proto) {
242     if (proto == null || !proto.hasClosed()) return false;
243     return proto.getClosed();
244   }
245 
246   /**
247    * A utility to build a GetServerInfoResponse.
248    *
249    * @param serverName
250    * @param webuiPort
251    * @return the response
252    */
253   public static GetServerInfoResponse buildGetServerInfoResponse(
254       final ServerName serverName, final int webuiPort) {
255     GetServerInfoResponse.Builder builder = GetServerInfoResponse.newBuilder();
256     ServerInfo.Builder serverInfoBuilder = ServerInfo.newBuilder();
257     serverInfoBuilder.setServerName(ProtobufUtil.toServerName(serverName));
258     if (webuiPort >= 0) {
259       serverInfoBuilder.setWebuiPort(webuiPort);
260     }
261     builder.setServerInfo(serverInfoBuilder.build());
262     return builder.build();
263   }
264 
265   /**
266    * A utility to build a GetOnlineRegionResponse.
267    *
268    * @param regions
269    * @return the response
270    */
271   public static GetOnlineRegionResponse buildGetOnlineRegionResponse(
272       final List<HRegionInfo> regions) {
273     GetOnlineRegionResponse.Builder builder = GetOnlineRegionResponse.newBuilder();
274     for (HRegionInfo region: regions) {
275       builder.addRegionInfo(HRegionInfo.convert(region));
276     }
277     return builder.build();
278   }
279 
280   /**
281    * Creates a response for the catalog scan request
282    * @return A RunCatalogScanResponse
283    */
284   public static RunCatalogScanResponse buildRunCatalogScanResponse(int numCleaned) {
285     return RunCatalogScanResponse.newBuilder().setScanResult(numCleaned).build();
286   }
287 
288   /**
289    * Creates a response for the catalog scan request
290    * @return A EnableCatalogJanitorResponse
291    */
292   public static EnableCatalogJanitorResponse buildEnableCatalogJanitorResponse(boolean prevValue) {
293     return EnableCatalogJanitorResponse.newBuilder().setPrevValue(prevValue).build();
294   }
295 
296 // End utilities for Admin
297 
298   /**
299    * Creates a response for the last flushed sequence Id request
300    * @return A GetLastFlushedSequenceIdResponse
301    */
302   public static GetLastFlushedSequenceIdResponse buildGetLastFlushedSequenceIdResponse(
303       long seqId) {
304     return GetLastFlushedSequenceIdResponse.newBuilder().setLastFlushedSequenceId(seqId).build();
305   }
306 
307   /**
308    * Stores an exception encountered during RPC invocation so it can be passed back
309    * through to the client.
310    * @param controller the controller instance provided by the client when calling the service
311    * @param ioe the exception encountered
312    */
313   public static void setControllerException(RpcController controller, IOException ioe) {
314     if (controller != null) {
315       if (controller instanceof ServerRpcController) {
316         ((ServerRpcController)controller).setFailedOn(ioe);
317       } else {
318         controller.setFailed(StringUtils.stringifyException(ioe));
319       }
320     }
321   }
322 
323   /**
324    * Create Results from the cells using the cells meta data. 
325    * @param cellScanner
326    * @param response
327    * @return results
328    */
329   public static Result[] getResults(CellScanner cellScanner, ScanResponse response)
330       throws IOException {
331     if (response == null) return null;
332     // If cellscanner, then the number of Results to return is the count of elements in the
333     // cellsPerResult list.  Otherwise, it is how many results are embedded inside the response.
334     int noOfResults = cellScanner != null?
335       response.getCellsPerResultCount(): response.getResultsCount();
336     Result[] results = new Result[noOfResults];
337     for (int i = 0; i < noOfResults; i++) {
338       if (cellScanner != null) {
339         // Cells are out in cellblocks.  Group them up again as Results.  How many to read at a
340         // time will be found in getCellsLength -- length here is how many Cells in the i'th Result
341         int noOfCells = response.getCellsPerResult(i);
342         List<Cell> cells = new ArrayList<Cell>(noOfCells);
343         for (int j = 0; j < noOfCells; j++) {
344           try {
345             if (cellScanner.advance() == false) {
346               // We are not able to retrieve the exact number of cells which ResultCellMeta says us.
347               // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
348               // same scanner will result in OutOfOrderScannerNextException
349               String msg = "Results sent from server=" + noOfResults + ". But only got " + i
350                 + " results completely at client. Resetting the scanner to scan again.";
351               LOG.error(msg);
352               throw new DoNotRetryIOException(msg);
353             }
354           } catch (IOException ioe) {
355             // We are getting IOE while retrieving the cells for Results.
356             // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
357             // same scanner will result in OutOfOrderScannerNextException
358             LOG.error("Exception while reading cells from result."
359               + "Resetting the scanner to scan again.", ioe);
360             throw new DoNotRetryIOException("Resetting the scanner.", ioe);
361           }
362           cells.add(cellScanner.current());
363         }
364         results[i] = Result.create(cells, null, response.getStale());
365       } else {
366         // Result is pure pb.
367         results[i] = ProtobufUtil.toResult(response.getResults(i));
368       }
369     }
370     return results;
371   }
372 }