View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.protobuf;
19  
20  import java.io.IOException;
21  import java.util.ArrayList;
22  import java.util.List;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.hbase.Cell;
27  import org.apache.hadoop.hbase.CellScanner;
28  import org.apache.hadoop.hbase.DoNotRetryIOException;
29  import org.apache.hadoop.hbase.HRegionInfo;
30  import org.apache.hadoop.hbase.ServerName;
31  import org.apache.hadoop.hbase.classification.InterfaceAudience;
32  import org.apache.hadoop.hbase.client.Result;
33  import org.apache.hadoop.hbase.ipc.ServerRpcController;
34  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.GetUserPermissionsResponse;
35  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
36  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
37  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
38  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
39  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse;
40  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
41  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
42  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
43  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
44  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
45  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
46  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
47  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
48  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
49  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
50  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorResponse;
51  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanResponse;
52  import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
53  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
54  import org.apache.hadoop.hbase.security.access.UserPermission;
55  import org.apache.hadoop.util.StringUtils;
56  
57  import com.google.protobuf.ByteString;
58  import com.google.protobuf.RpcController;
59  
60  /**
61   * Helper utility to build protocol buffer responses,
62   * or retrieve data from protocol buffer responses.
63   */
64  @InterfaceAudience.Private
65  public final class ResponseConverter {
66    public static final Log LOG = LogFactory.getLog(ResponseConverter.class);
67  
68    private ResponseConverter() {
69    }
70  
71  // Start utilities for Client
72  
73    /**
74     * Get the results from a protocol buffer MultiResponse
75     *
76     * @param request the protocol buffer MultiResponse to convert
77     * @param cells Cells to go with the passed in <code>proto</code>.  Can be null.
78     * @return the results that were in the MultiResponse (a Result or an Exception).
79     * @throws IOException
80     */
81    public static org.apache.hadoop.hbase.client.MultiResponse getResults(final MultiRequest request,
82        final MultiResponse response, final CellScanner cells)
83    throws IOException {
84      int requestRegionActionCount = request.getRegionActionCount();
85      int responseRegionActionResultCount = response.getRegionActionResultCount();
86      if (requestRegionActionCount != responseRegionActionResultCount) {
87        throw new IllegalStateException("Request mutation count=" + responseRegionActionResultCount +
88            " does not match response mutation result count=" + responseRegionActionResultCount);
89      }
90  
91      org.apache.hadoop.hbase.client.MultiResponse results =
92        new org.apache.hadoop.hbase.client.MultiResponse();
93  
94      for (int i = 0; i < responseRegionActionResultCount; i++) {
95        RegionAction actions = request.getRegionAction(i);
96        RegionActionResult actionResult = response.getRegionActionResult(i);
97        HBaseProtos.RegionSpecifier rs = actions.getRegion();
98        if (rs.hasType() &&
99            (rs.getType() != HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME)){
100         throw new IllegalArgumentException(
101             "We support only encoded types for protobuf multi response.");
102       }
103       byte[] regionName = rs.getValue().toByteArray();
104 
105       if (actionResult.hasException()) {
106         Throwable regionException =  ProtobufUtil.toException(actionResult.getException());
107         results.addException(regionName, regionException);
108         continue;
109       }
110 
111       if (actions.getActionCount() != actionResult.getResultOrExceptionCount()) {
112         throw new IllegalStateException("actions.getActionCount=" + actions.getActionCount() +
113             ", actionResult.getResultOrExceptionCount=" +
114             actionResult.getResultOrExceptionCount() + " for region " + actions.getRegion());
115       }
116 
117       for (ResultOrException roe : actionResult.getResultOrExceptionList()) {
118         if (roe.hasException()) {
119           results.add(regionName, roe.getIndex(), ProtobufUtil.toException(roe.getException()));
120         } else if (roe.hasResult()) {
121           results.add(regionName, roe.getIndex(), ProtobufUtil.toResult(roe.getResult(), cells));
122         } else if (roe.hasServiceResult()) {
123           results.add(regionName, roe.getIndex(), roe.getServiceResult());
124         } else {
125           // no result & no exception. Unexpected.
126           throw new IllegalStateException("No result & no exception roe=" + roe +
127               " for region " + actions.getRegion());
128         }
129       }
130     }
131 
132     return results;
133   }
134 
135   /**
136    * Wrap a throwable to an action result.
137    *
138    * @param t
139    * @return an action result builder
140    */
141   public static ResultOrException.Builder buildActionResult(final Throwable t) {
142     ResultOrException.Builder builder = ResultOrException.newBuilder();
143     if (t != null) builder.setException(buildException(t));
144     return builder;
145   }
146 
147   /**
148    * Wrap a throwable to an action result.
149    *
150    * @param r
151    * @return an action result builder
152    */
153   public static ResultOrException.Builder buildActionResult(final ClientProtos.Result r) {
154     ResultOrException.Builder builder = ResultOrException.newBuilder();
155     if (r != null) builder.setResult(r);
156     return builder;
157   }
158 
159   /**
160    * @param t
161    * @return NameValuePair of the exception name to stringified version os exception.
162    */
163   public static NameBytesPair buildException(final Throwable t) {
164     NameBytesPair.Builder parameterBuilder = NameBytesPair.newBuilder();
165     parameterBuilder.setName(t.getClass().getName());
166     parameterBuilder.setValue(
167       ByteString.copyFromUtf8(StringUtils.stringifyException(t)));
168     return parameterBuilder.build();
169   }
170 
171   /**
172    * Converts the permissions list into a protocol buffer GetUserPermissionsResponse
173    */
174   public static GetUserPermissionsResponse buildGetUserPermissionsResponse(
175       final List<UserPermission> permissions) {
176     GetUserPermissionsResponse.Builder builder = GetUserPermissionsResponse.newBuilder();
177     for (UserPermission perm : permissions) {
178       builder.addUserPermission(ProtobufUtil.toUserPermission(perm));
179     }
180     return builder.build();
181   }
182 
183 // End utilities for Client
184 // Start utilities for Admin
185 
186   /**
187    * Get the list of regions to flush from a RollLogWriterResponse
188    *
189    * @param proto the RollLogWriterResponse
190    * @return the the list of regions to flush
191    */
192   public static byte[][] getRegions(final RollWALWriterResponse proto) {
193     if (proto == null || proto.getRegionToFlushCount() == 0) return null;
194     List<byte[]> regions = new ArrayList<byte[]>();
195     for (ByteString region: proto.getRegionToFlushList()) {
196       regions.add(region.toByteArray());
197     }
198     return (byte[][])regions.toArray();
199   }
200 
201   /**
202    * Get the list of region info from a GetOnlineRegionResponse
203    *
204    * @param proto the GetOnlineRegionResponse
205    * @return the list of region info
206    */
207   public static List<HRegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
208     if (proto == null || proto.getRegionInfoCount() == 0) return null;
209     return ProtobufUtil.getRegionInfos(proto);
210   }
211 
212   /**
213    * Get the region opening state from a OpenRegionResponse
214    *
215    * @param proto the OpenRegionResponse
216    * @return the region opening state
217    */
218   public static RegionOpeningState getRegionOpeningState
219       (final OpenRegionResponse proto) {
220     if (proto == null || proto.getOpeningStateCount() != 1) return null;
221     return RegionOpeningState.valueOf(
222       proto.getOpeningState(0).name());
223   }
224 
225   /**
226    * Get a list of region opening state from a OpenRegionResponse
227    * 
228    * @param proto the OpenRegionResponse
229    * @return the list of region opening state
230    */
231   public static List<RegionOpeningState> getRegionOpeningStateList(
232       final OpenRegionResponse proto) {
233     if (proto == null) return null;
234     List<RegionOpeningState> regionOpeningStates = new ArrayList<RegionOpeningState>();
235     for (int i = 0; i < proto.getOpeningStateCount(); i++) {
236       regionOpeningStates.add(RegionOpeningState.valueOf(
237           proto.getOpeningState(i).name()));
238     }
239     return regionOpeningStates;
240   }
241 
242   /**
243    * Check if the region is closed from a CloseRegionResponse
244    *
245    * @param proto the CloseRegionResponse
246    * @return the region close state
247    */
248   public static boolean isClosed
249       (final CloseRegionResponse proto) {
250     if (proto == null || !proto.hasClosed()) return false;
251     return proto.getClosed();
252   }
253 
254   /**
255    * A utility to build a GetServerInfoResponse.
256    *
257    * @param serverName
258    * @param webuiPort
259    * @return the response
260    */
261   public static GetServerInfoResponse buildGetServerInfoResponse(
262       final ServerName serverName, final int webuiPort) {
263     GetServerInfoResponse.Builder builder = GetServerInfoResponse.newBuilder();
264     ServerInfo.Builder serverInfoBuilder = ServerInfo.newBuilder();
265     serverInfoBuilder.setServerName(ProtobufUtil.toServerName(serverName));
266     if (webuiPort >= 0) {
267       serverInfoBuilder.setWebuiPort(webuiPort);
268     }
269     builder.setServerInfo(serverInfoBuilder.build());
270     return builder.build();
271   }
272 
273   /**
274    * A utility to build a GetOnlineRegionResponse.
275    *
276    * @param regions
277    * @return the response
278    */
279   public static GetOnlineRegionResponse buildGetOnlineRegionResponse(
280       final List<HRegionInfo> regions) {
281     GetOnlineRegionResponse.Builder builder = GetOnlineRegionResponse.newBuilder();
282     for (HRegionInfo region: regions) {
283       builder.addRegionInfo(HRegionInfo.convert(region));
284     }
285     return builder.build();
286   }
287 
288   /**
289    * Creates a response for the catalog scan request
290    * @return A RunCatalogScanResponse
291    */
292   public static RunCatalogScanResponse buildRunCatalogScanResponse(int numCleaned) {
293     return RunCatalogScanResponse.newBuilder().setScanResult(numCleaned).build();
294   }
295 
296   /**
297    * Creates a response for the catalog scan request
298    * @return A EnableCatalogJanitorResponse
299    */
300   public static EnableCatalogJanitorResponse buildEnableCatalogJanitorResponse(boolean prevValue) {
301     return EnableCatalogJanitorResponse.newBuilder().setPrevValue(prevValue).build();
302   }
303 
304 // End utilities for Admin
305 
306   /**
307    * Creates a response for the last flushed sequence Id request
308    * @return A GetLastFlushedSequenceIdResponse
309    */
310   public static GetLastFlushedSequenceIdResponse buildGetLastFlushedSequenceIdResponse(
311       long seqId) {
312     return GetLastFlushedSequenceIdResponse.newBuilder().setLastFlushedSequenceId(seqId).build();
313   }
314 
315   /**
316    * Stores an exception encountered during RPC invocation so it can be passed back
317    * through to the client.
318    * @param controller the controller instance provided by the client when calling the service
319    * @param ioe the exception encountered
320    */
321   public static void setControllerException(RpcController controller, IOException ioe) {
322     if (controller != null) {
323       if (controller instanceof ServerRpcController) {
324         ((ServerRpcController)controller).setFailedOn(ioe);
325       } else {
326         controller.setFailed(StringUtils.stringifyException(ioe));
327       }
328     }
329   }
330 
331   /**
332    * Create Results from the cells using the cells meta data. 
333    * @param cellScanner
334    * @param response
335    * @return results
336    */
337   public static Result[] getResults(CellScanner cellScanner, ScanResponse response)
338       throws IOException {
339     if (response == null) return null;
340     // If cellscanner, then the number of Results to return is the count of elements in the
341     // cellsPerResult list.  Otherwise, it is how many results are embedded inside the response.
342     int noOfResults = cellScanner != null?
343       response.getCellsPerResultCount(): response.getResultsCount();
344     Result[] results = new Result[noOfResults];
345     for (int i = 0; i < noOfResults; i++) {
346       if (cellScanner != null) {
347         // Cells are out in cellblocks.  Group them up again as Results.  How many to read at a
348         // time will be found in getCellsLength -- length here is how many Cells in the i'th Result
349         int noOfCells = response.getCellsPerResult(i);
350         List<Cell> cells = new ArrayList<Cell>(noOfCells);
351         for (int j = 0; j < noOfCells; j++) {
352           try {
353             if (cellScanner.advance() == false) {
354               // We are not able to retrieve the exact number of cells which ResultCellMeta says us.
355               // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
356               // same scanner will result in OutOfOrderScannerNextException
357               String msg = "Results sent from server=" + noOfResults + ". But only got " + i
358                 + " results completely at client. Resetting the scanner to scan again.";
359               LOG.error(msg);
360               throw new DoNotRetryIOException(msg);
361             }
362           } catch (IOException ioe) {
363             // We are getting IOE while retrieving the cells for Results.
364             // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
365             // same scanner will result in OutOfOrderScannerNextException
366             LOG.error("Exception while reading cells from result."
367               + "Resetting the scanner to scan again.", ioe);
368             throw new DoNotRetryIOException("Resetting the scanner.", ioe);
369           }
370           cells.add(cellScanner.current());
371         }
372         results[i] = Result.create(cells, null, response.getStale());
373       } else {
374         // Result is pure pb.
375         results[i] = ProtobufUtil.toResult(response.getResults(i));
376       }
377     }
378     return results;
379   }
380 }