001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.ipc;
019
020import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION;
021
022import com.google.errorprone.annotations.RestrictedApi;
023import java.io.IOException;
024import java.net.InetAddress;
025import java.net.InetSocketAddress;
026import java.nio.ByteBuffer;
027import java.nio.channels.ReadableByteChannel;
028import java.nio.channels.WritableByteChannel;
029import java.util.Collections;
030import java.util.HashMap;
031import java.util.List;
032import java.util.Locale;
033import java.util.Map;
034import java.util.Optional;
035import java.util.concurrent.atomic.LongAdder;
036import org.apache.commons.lang3.StringUtils;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.hbase.CallQueueTooBigException;
039import org.apache.hadoop.hbase.CellScanner;
040import org.apache.hadoop.hbase.DoNotRetryIOException;
041import org.apache.hadoop.hbase.ExtendedCellScanner;
042import org.apache.hadoop.hbase.HConstants;
043import org.apache.hadoop.hbase.Server;
044import org.apache.hadoop.hbase.conf.ConfigurationObserver;
045import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
046import org.apache.hadoop.hbase.io.ByteBuffAllocator;
047import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
048import org.apache.hadoop.hbase.monitoring.TaskMonitor;
049import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics;
050import org.apache.hadoop.hbase.namequeues.NamedQueueRecorder;
051import org.apache.hadoop.hbase.namequeues.RpcLogDetails;
052import org.apache.hadoop.hbase.regionserver.RSRpcServices;
053import org.apache.hadoop.hbase.security.HBasePolicyProvider;
054import org.apache.hadoop.hbase.security.SaslUtil;
055import org.apache.hadoop.hbase.security.SaslUtil.QualityOfProtection;
056import org.apache.hadoop.hbase.security.User;
057import org.apache.hadoop.hbase.security.UserProvider;
058import org.apache.hadoop.hbase.security.token.AuthenticationTokenSecretManager;
059import org.apache.hadoop.hbase.util.CoprocessorConfigurationUtil;
060import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
061import org.apache.hadoop.hbase.util.GsonUtil;
062import org.apache.hadoop.hbase.util.Pair;
063import org.apache.hadoop.security.UserGroupInformation;
064import org.apache.hadoop.security.authorize.AuthorizationException;
065import org.apache.hadoop.security.authorize.PolicyProvider;
066import org.apache.hadoop.security.authorize.ProxyUsers;
067import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
068import org.apache.hadoop.security.token.SecretManager;
069import org.apache.hadoop.security.token.TokenIdentifier;
070import org.apache.yetus.audience.InterfaceAudience;
071import org.slf4j.Logger;
072import org.slf4j.LoggerFactory;
073
074import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
075import org.apache.hbase.thirdparty.com.google.gson.Gson;
076import org.apache.hbase.thirdparty.com.google.protobuf.BlockingService;
077import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors.MethodDescriptor;
078import org.apache.hbase.thirdparty.com.google.protobuf.Message;
079import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
080import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;
081
082import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
083import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
084import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.ConnectionHeader;
085
086/**
087 * An RPC server that hosts protobuf described Services.
088 */
089@InterfaceAudience.Private
090public abstract class RpcServer implements RpcServerInterface, ConfigurationObserver {
091  // LOG is being used in CallRunner and the log level is being changed in tests
092  public static final Logger LOG = LoggerFactory.getLogger(RpcServer.class);
093  protected static final CallQueueTooBigException CALL_QUEUE_TOO_BIG_EXCEPTION =
094    new CallQueueTooBigException();
095
096  private static final String MULTI_GETS = "multi.gets";
097  private static final String MULTI_MUTATIONS = "multi.mutations";
098  private static final String MULTI_SERVICE_CALLS = "multi.service_calls";
099
100  private final boolean authorize;
101  private volatile boolean isOnlineLogProviderEnabled;
102  protected boolean isSecurityEnabled;
103
104  public static final byte CURRENT_VERSION = 0;
105
106  /**
107   * Whether we allow a fallback to SIMPLE auth for insecure clients when security is enabled.
108   */
109  public static final String FALLBACK_TO_INSECURE_CLIENT_AUTH =
110    "hbase.ipc.server.fallback-to-simple-auth-allowed";
111
112  /**
113   * How many calls/handler are allowed in the queue.
114   */
115  protected static final int DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER = 10;
116
117  protected final CellBlockBuilder cellBlockBuilder;
118
119  protected static final String AUTH_FAILED_FOR = "Auth failed for ";
120  protected static final String AUTH_SUCCESSFUL_FOR = "Auth successful for ";
121  protected static final Logger AUDITLOG =
122    LoggerFactory.getLogger("SecurityLogger." + Server.class.getName());
123  protected SecretManager<TokenIdentifier> secretManager;
124  protected final Map<String, String> saslProps;
125  protected final String serverPrincipal;
126
127  protected ServiceAuthorizationManager authManager;
128
129  /**
130   * This is set to Call object before Handler invokes an RPC and ybdie after the call returns.
131   */
132  protected static final ThreadLocal<RpcCall> CurCall = new ThreadLocal<>();
133
134  /** Keeps MonitoredRPCHandler per handler thread. */
135  protected static final ThreadLocal<MonitoredRPCHandler> MONITORED_RPC = new ThreadLocal<>();
136
137  protected final InetSocketAddress bindAddress;
138
139  protected MetricsHBaseServer metrics;
140
141  protected final Configuration conf;
142
143  /**
144   * Maximum size in bytes of the currently queued and running Calls. If a new Call puts us over
145   * this size, then we will reject the call (after parsing it though). It will go back to the
146   * client and client will retry. Set this size with "hbase.ipc.server.max.callqueue.size". The
147   * call queue size gets incremented after we parse a call and before we add it to the queue of
148   * calls for the scheduler to use. It get decremented after we have 'run' the Call. The current
149   * size is kept in {@link #callQueueSizeInBytes}.
150   * @see #callQueueSizeInBytes
151   * @see #DEFAULT_MAX_CALLQUEUE_SIZE
152   */
153  protected final long maxQueueSizeInBytes;
154  protected static final int DEFAULT_MAX_CALLQUEUE_SIZE = 1024 * 1024 * 1024;
155
156  /**
157   * This is a running count of the size in bytes of all outstanding calls whether currently
158   * executing or queued waiting to be run.
159   */
160  protected final LongAdder callQueueSizeInBytes = new LongAdder();
161
162  protected final boolean tcpNoDelay; // if T then disable Nagle's Algorithm
163  protected final boolean tcpKeepAlive; // if T then use keepalives
164
165  /**
166   * This flag is used to indicate to sub threads when they should go down. When we call
167   * {@link #start()}, all threads started will consult this flag on whether they should keep going.
168   * It is set to false when {@link #stop()} is called.
169   */
170  volatile boolean running = true;
171
172  /**
173   * This flag is set to true after all threads are up and 'running' and the server is then opened
174   * for business by the call to {@link #start()}.
175   */
176  volatile boolean started = false;
177
178  protected AuthenticationTokenSecretManager authTokenSecretMgr = null;
179
180  protected HBaseRPCErrorHandler errorHandler = null;
181
182  public static final String MAX_REQUEST_SIZE = "hbase.ipc.max.request.size";
183
184  protected static final String WARN_RESPONSE_TIME = "hbase.ipc.warn.response.time";
185  protected static final String WARN_RESPONSE_SIZE = "hbase.ipc.warn.response.size";
186  protected static final String WARN_SCAN_RESPONSE_TIME = "hbase.ipc.warn.response.time.scan";
187  protected static final String WARN_SCAN_RESPONSE_SIZE = "hbase.ipc.warn.response.size.scan";
188
189  /**
190   * Minimum allowable timeout (in milliseconds) in rpc request's header. This configuration exists
191   * to prevent the rpc service regarding this request as timeout immediately.
192   */
193  protected static final String MIN_CLIENT_REQUEST_TIMEOUT = "hbase.ipc.min.client.request.timeout";
194  protected static final int DEFAULT_MIN_CLIENT_REQUEST_TIMEOUT = 20;
195
196  /** Default value for above params */
197  public static final int DEFAULT_MAX_REQUEST_SIZE = DEFAULT_MAX_CALLQUEUE_SIZE / 4; // 256M
198  protected static final int DEFAULT_WARN_RESPONSE_TIME = 10000; // milliseconds
199  protected static final int DEFAULT_WARN_RESPONSE_SIZE = 100 * 1024 * 1024;
200
201  protected static final int DEFAULT_TRACE_LOG_MAX_LENGTH = 1000;
202  protected static final String TRACE_LOG_MAX_LENGTH = "hbase.ipc.trace.log.max.length";
203  protected static final String KEY_WORD_TRUNCATED = " <TRUNCATED>";
204
205  protected static final Gson GSON = GsonUtil.createGsonWithDisableHtmlEscaping().create();
206
207  protected final int maxRequestSize;
208  protected volatile int warnResponseTime;
209  protected volatile int warnResponseSize;
210  protected volatile int warnScanResponseTime;
211  protected volatile int warnScanResponseSize;
212
213  protected final int minClientRequestTimeout;
214
215  protected final Server server;
216  protected final List<BlockingServiceAndInterface> services;
217
218  protected final RpcScheduler scheduler;
219
220  protected final UserProvider userProvider;
221
222  protected final ByteBuffAllocator bbAllocator;
223
224  protected volatile boolean allowFallbackToSimpleAuth;
225
226  volatile RpcCoprocessorHost cpHost;
227
228  /**
229   * Used to get details for scan with a scanner_id<br/>
230   * TODO try to figure out a better way and remove reference from regionserver package later.
231   */
232  private RSRpcServices rsRpcServices;
233
234  /**
235   * Use to add online slowlog responses
236   */
237  private NamedQueueRecorder namedQueueRecorder;
238
239  @FunctionalInterface
240  protected interface CallCleanup {
241    void run();
242  }
243
244  /**
245   * Datastructure for passing a {@link BlockingService} and its associated class of protobuf
246   * service interface. For example, a server that fielded what is defined in the client protobuf
247   * service would pass in an implementation of the client blocking service and then its
248   * ClientService.BlockingInterface.class. Used checking connection setup.
249   */
250  public static class BlockingServiceAndInterface {
251    private final BlockingService service;
252    private final Class<?> serviceInterface;
253
254    public BlockingServiceAndInterface(final BlockingService service,
255      final Class<?> serviceInterface) {
256      this.service = service;
257      this.serviceInterface = serviceInterface;
258    }
259
260    public Class<?> getServiceInterface() {
261      return this.serviceInterface;
262    }
263
264    public BlockingService getBlockingService() {
265      return this.service;
266    }
267  }
268
269  /**
270   * Constructs a server listening on the named port and address.
271   * @param server           hosting instance of {@link Server}. We will do authentications if an
272   *                         instance else pass null for no authentication check.
273   * @param name             Used keying this rpc servers' metrics and for naming the Listener
274   *                         thread.
275   * @param services         A list of services.
276   * @param bindAddress      Where to listen
277   * @param reservoirEnabled Enable ByteBufferPool or not.
278   */
279  public RpcServer(final Server server, final String name,
280    final List<BlockingServiceAndInterface> services, final InetSocketAddress bindAddress,
281    Configuration conf, RpcScheduler scheduler, boolean reservoirEnabled) throws IOException {
282    this.bbAllocator = ByteBuffAllocator.create(conf, reservoirEnabled);
283    this.server = server;
284    this.services = services;
285    this.bindAddress = bindAddress;
286    this.conf = conf;
287    // See declaration above for documentation on what this size is.
288    this.maxQueueSizeInBytes =
289      this.conf.getLong("hbase.ipc.server.max.callqueue.size", DEFAULT_MAX_CALLQUEUE_SIZE);
290
291    this.warnResponseTime = getWarnResponseTime(conf);
292    this.warnResponseSize = getWarnResponseSize(conf);
293    this.warnScanResponseTime = getWarnScanResponseTime(conf);
294    this.warnScanResponseSize = getWarnScanResponseSize(conf);
295    this.minClientRequestTimeout =
296      conf.getInt(MIN_CLIENT_REQUEST_TIMEOUT, DEFAULT_MIN_CLIENT_REQUEST_TIMEOUT);
297    this.maxRequestSize = conf.getInt(MAX_REQUEST_SIZE, DEFAULT_MAX_REQUEST_SIZE);
298
299    this.metrics = new MetricsHBaseServer(name, new MetricsHBaseServerWrapperImpl(this));
300    this.tcpNoDelay = conf.getBoolean("hbase.ipc.server.tcpnodelay", true);
301    this.tcpKeepAlive = conf.getBoolean("hbase.ipc.server.tcpkeepalive", true);
302
303    this.cellBlockBuilder = new CellBlockBuilder(conf);
304
305    this.authorize = conf.getBoolean(HADOOP_SECURITY_AUTHORIZATION, false);
306    this.userProvider = UserProvider.instantiate(conf);
307    this.isSecurityEnabled = userProvider.isHBaseSecurityEnabled();
308    if (isSecurityEnabled) {
309      saslProps = SaslUtil.initSaslProperties(conf.get("hbase.rpc.protection",
310        QualityOfProtection.AUTHENTICATION.name().toLowerCase(Locale.ROOT)));
311      serverPrincipal = Preconditions.checkNotNull(userProvider.getCurrentUserName(),
312        "can not get current user name when security is enabled");
313    } else {
314      saslProps = Collections.emptyMap();
315      serverPrincipal = HConstants.EMPTY_STRING;
316    }
317
318    this.isOnlineLogProviderEnabled = getIsOnlineLogProviderEnabled(conf);
319    this.scheduler = scheduler;
320
321    initializeCoprocessorHost(getConf());
322  }
323
324  @Override
325  public void onConfigurationChange(Configuration newConf) {
326    initReconfigurable(newConf);
327    if (scheduler instanceof ConfigurationObserver) {
328      ((ConfigurationObserver) scheduler).onConfigurationChange(newConf);
329    }
330    if (authorize) {
331      refreshAuthManager(newConf, new HBasePolicyProvider());
332    }
333    refreshSlowLogConfiguration(newConf);
334    if (
335      CoprocessorConfigurationUtil.checkConfigurationChange(getConf(), newConf,
336        CoprocessorHost.RPC_COPROCESSOR_CONF_KEY)
337    ) {
338      LOG.info("Update the RPC coprocessor(s) because the configuration has changed");
339      initializeCoprocessorHost(newConf);
340    }
341  }
342
343  private void refreshSlowLogConfiguration(Configuration newConf) {
344    boolean newIsOnlineLogProviderEnabled = getIsOnlineLogProviderEnabled(newConf);
345    if (isOnlineLogProviderEnabled != newIsOnlineLogProviderEnabled) {
346      isOnlineLogProviderEnabled = newIsOnlineLogProviderEnabled;
347    }
348    int newWarnResponseTime = getWarnResponseTime(newConf);
349    if (warnResponseTime != newWarnResponseTime) {
350      warnResponseTime = newWarnResponseTime;
351    }
352    int newWarnResponseSize = getWarnResponseSize(newConf);
353    if (warnResponseSize != newWarnResponseSize) {
354      warnResponseSize = newWarnResponseSize;
355    }
356    int newWarnResponseTimeScan = getWarnScanResponseTime(newConf);
357    if (warnScanResponseTime != newWarnResponseTimeScan) {
358      warnScanResponseTime = newWarnResponseTimeScan;
359    }
360    int newWarnScanResponseSize = getWarnScanResponseSize(newConf);
361    if (warnScanResponseSize != newWarnScanResponseSize) {
362      warnScanResponseSize = newWarnScanResponseSize;
363    }
364  }
365
366  private static boolean getIsOnlineLogProviderEnabled(Configuration conf) {
367    return conf.getBoolean(HConstants.SLOW_LOG_BUFFER_ENABLED_KEY,
368      HConstants.DEFAULT_ONLINE_LOG_PROVIDER_ENABLED);
369  }
370
371  private static int getWarnResponseTime(Configuration conf) {
372    return conf.getInt(WARN_RESPONSE_TIME, DEFAULT_WARN_RESPONSE_TIME);
373  }
374
375  private static int getWarnResponseSize(Configuration conf) {
376    return conf.getInt(WARN_RESPONSE_SIZE, DEFAULT_WARN_RESPONSE_SIZE);
377  }
378
379  private static int getWarnScanResponseTime(Configuration conf) {
380    return conf.getInt(WARN_SCAN_RESPONSE_TIME, getWarnResponseTime(conf));
381  }
382
383  private static int getWarnScanResponseSize(Configuration conf) {
384    return conf.getInt(WARN_SCAN_RESPONSE_SIZE, getWarnResponseSize(conf));
385  }
386
387  protected void initReconfigurable(Configuration confToLoad) {
388    this.allowFallbackToSimpleAuth = confToLoad.getBoolean(FALLBACK_TO_INSECURE_CLIENT_AUTH, false);
389    if (isSecurityEnabled && allowFallbackToSimpleAuth) {
390      LOG.warn("********* WARNING! *********");
391      LOG.warn("This server is configured to allow connections from INSECURE clients");
392      LOG.warn("(" + FALLBACK_TO_INSECURE_CLIENT_AUTH + " = true).");
393      LOG.warn("While this option is enabled, client identities cannot be secured, and user");
394      LOG.warn("impersonation is possible!");
395      LOG.warn("For secure operation, please disable SIMPLE authentication as soon as possible,");
396      LOG.warn("by setting " + FALLBACK_TO_INSECURE_CLIENT_AUTH + " = false in hbase-site.xml");
397      LOG.warn("****************************");
398    }
399  }
400
401  Configuration getConf() {
402    return conf;
403  }
404
405  @Override
406  public boolean isStarted() {
407    return this.started;
408  }
409
410  @Override
411  public synchronized void refreshAuthManager(Configuration conf, PolicyProvider pp) {
412    // Ignore warnings that this should be accessed in a static way instead of via an instance;
413    // it'll break if you go via static route.
414    System.setProperty("hadoop.policy.file", "hbase-policy.xml");
415    this.authManager.refresh(conf, pp);
416    LOG.info("Refreshed hbase-policy.xml successfully");
417    ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
418    LOG.info("Refreshed super and proxy users successfully");
419  }
420
421  protected AuthenticationTokenSecretManager createSecretManager() {
422    if (!isSecurityEnabled) return null;
423    if (server == null) return null;
424    Configuration conf = server.getConfiguration();
425    long keyUpdateInterval = conf.getLong("hbase.auth.key.update.interval", 24 * 60 * 60 * 1000);
426    long maxAge = conf.getLong("hbase.auth.token.max.lifetime", 7 * 24 * 60 * 60 * 1000);
427    return new AuthenticationTokenSecretManager(conf, server.getZooKeeper(),
428      server.getServerName().toString(), keyUpdateInterval, maxAge);
429  }
430
431  public SecretManager<? extends TokenIdentifier> getSecretManager() {
432    return this.secretManager;
433  }
434
435  @SuppressWarnings("unchecked")
436  public void setSecretManager(SecretManager<? extends TokenIdentifier> secretManager) {
437    this.secretManager = (SecretManager<TokenIdentifier>) secretManager;
438  }
439
440  /**
441   * This is a server side method, which is invoked over RPC. On success the return response has
442   * protobuf response payload. On failure, the exception name and the stack trace are returned in
443   * the protobuf response.
444   */
445  @Override
446  public Pair<Message, ExtendedCellScanner> call(RpcCall call, MonitoredRPCHandler status)
447    throws IOException {
448    try {
449      MethodDescriptor md = call.getMethod();
450      Message param = call.getParam();
451      status.setRPC(md.getName(), new Object[] { param }, call.getReceiveTime());
452      // TODO: Review after we add in encoded data blocks.
453      status.setRPCPacket(param);
454      status.resume("Servicing call");
455      // get an instance of the method arg type
456      HBaseRpcController controller = new HBaseRpcControllerImpl(call.getCellScanner());
457      controller.setCallTimeout(call.getTimeout());
458      Message result = call.getService().callBlockingMethod(md, controller, param);
459      long receiveTime = call.getReceiveTime();
460      long startTime = call.getStartTime();
461      long endTime = EnvironmentEdgeManager.currentTime();
462      int processingTime = (int) (endTime - startTime);
463      int qTime = (int) (startTime - receiveTime);
464      int totalTime = (int) (endTime - receiveTime);
465      long fsReadTime = ThreadLocalServerSideScanMetrics.getFsReadTimeCounter().get();
466      if (LOG.isTraceEnabled()) {
467        LOG.trace(
468          "{}, response: {}, receiveTime: {}, queueTime: {}, processingTime: {}, "
469            + "totalTime: {}, fsReadTime: {}",
470          CurCall.get().toString(), TextFormat.shortDebugString(result),
471          CurCall.get().getReceiveTime(), qTime, processingTime, totalTime, fsReadTime);
472      }
473      // Use the raw request call size for now.
474      long requestSize = call.getSize();
475      long responseSize = result.getSerializedSize();
476      long responseBlockSize = call.getBlockBytesScanned();
477      if (call.isClientCellBlockSupported()) {
478        // Include the payload size in HBaseRpcController
479        responseSize += call.getResponseCellSize();
480      }
481
482      metrics.dequeuedCall(qTime);
483      metrics.processedCall(processingTime);
484      metrics.totalCall(totalTime);
485      metrics.receivedRequest(requestSize);
486      metrics.sentResponse(responseSize);
487      // log any RPC responses that are slower than the configured warn
488      // response time or larger than configured warning size
489      boolean tooSlow = isTooSlow(call, processingTime);
490      boolean tooLarge = isTooLarge(call, responseSize, responseBlockSize);
491      if (tooSlow || tooLarge) {
492        final String userName = call.getRequestUserName().orElse(StringUtils.EMPTY);
493        // when tagging, we let TooLarge trump TooSmall to keep output simple
494        // note that large responses will often also be slow.
495        logResponse(param, md.getName(), md.getName() + "(" + param.getClass().getName() + ")",
496          tooLarge, tooSlow, status.getClient(), startTime, processingTime, qTime, responseSize,
497          responseBlockSize, fsReadTime, userName);
498        if (this.namedQueueRecorder != null && this.isOnlineLogProviderEnabled) {
499          // send logs to ring buffer owned by slowLogRecorder
500          final String className =
501            server == null ? StringUtils.EMPTY : server.getClass().getSimpleName();
502          this.namedQueueRecorder.addRecord(new RpcLogDetails(call, param, status.getClient(),
503            responseSize, responseBlockSize, fsReadTime, className, tooSlow, tooLarge));
504        }
505      }
506      return new Pair<>(result, controller.cellScanner());
507    } catch (Throwable e) {
508      // The above callBlockingMethod will always return a SE. Strip the SE wrapper before
509      // putting it on the wire. Its needed to adhere to the pb Service Interface but we don't
510      // need to pass it over the wire.
511      if (e instanceof ServiceException) {
512        if (e.getCause() == null) {
513          LOG.debug("Caught a ServiceException with null cause", e);
514        } else {
515          e = e.getCause();
516        }
517      }
518
519      // increment the number of requests that were exceptions.
520      metrics.exception(e);
521
522      if (e instanceof LinkageError) throw new DoNotRetryIOException(e);
523      if (e instanceof IOException) throw (IOException) e;
524      LOG.error("Unexpected throwable object ", e);
525      throw new IOException(e.getMessage(), e);
526    }
527  }
528
529  /**
530   * Logs an RPC response to the LOG file, producing valid JSON objects for client Operations.
531   * @param param             The parameters received in the call.
532   * @param methodName        The name of the method invoked
533   * @param call              The string representation of the call
534   * @param tooLarge          To indicate if the event is tooLarge
535   * @param tooSlow           To indicate if the event is tooSlow
536   * @param clientAddress     The address of the client who made this call.
537   * @param startTime         The time that the call was initiated, in ms.
538   * @param processingTime    The duration that the call took to run, in ms.
539   * @param qTime             The duration that the call spent on the queue prior to being
540   *                          initiated, in ms.
541   * @param responseSize      The size in bytes of the response buffer.
542   * @param blockBytesScanned The size of block bytes scanned to retrieve the response.
543   * @param userName          UserName of the current RPC Call
544   */
545  void logResponse(Message param, String methodName, String call, boolean tooLarge, boolean tooSlow,
546    String clientAddress, long startTime, int processingTime, int qTime, long responseSize,
547    long blockBytesScanned, long fsReadTime, String userName) {
548    final String className = server == null ? StringUtils.EMPTY : server.getClass().getSimpleName();
549    // base information that is reported regardless of type of call
550    Map<String, Object> responseInfo = new HashMap<>();
551    responseInfo.put("starttimems", startTime);
552    responseInfo.put("processingtimems", processingTime);
553    responseInfo.put("queuetimems", qTime);
554    responseInfo.put("responsesize", responseSize);
555    responseInfo.put("blockbytesscanned", blockBytesScanned);
556    responseInfo.put("fsreadtime", fsReadTime);
557    responseInfo.put("client", clientAddress);
558    responseInfo.put("class", className);
559    responseInfo.put("method", methodName);
560    responseInfo.put("call", call);
561    // The params could be really big, make sure they don't kill us at WARN
562    String stringifiedParam = ProtobufUtil.getShortTextFormat(param);
563    if (stringifiedParam.length() > 150) {
564      // Truncate to 1000 chars if TRACE is on, else to 150 chars
565      stringifiedParam = truncateTraceLog(stringifiedParam);
566    }
567    responseInfo.put("param", stringifiedParam);
568    if (param instanceof ClientProtos.ScanRequest && rsRpcServices != null) {
569      ClientProtos.ScanRequest request = ((ClientProtos.ScanRequest) param);
570      String scanDetails;
571      if (request.hasScannerId()) {
572        long scannerId = request.getScannerId();
573        scanDetails = rsRpcServices.getScanDetailsWithId(scannerId);
574      } else {
575        scanDetails = rsRpcServices.getScanDetailsWithRequest(request);
576      }
577      if (scanDetails != null) {
578        responseInfo.put("scandetails", scanDetails);
579      }
580    }
581    if (param instanceof ClientProtos.MultiRequest) {
582      int numGets = 0;
583      int numMutations = 0;
584      int numServiceCalls = 0;
585      ClientProtos.MultiRequest multi = (ClientProtos.MultiRequest) param;
586      for (ClientProtos.RegionAction regionAction : multi.getRegionActionList()) {
587        for (ClientProtos.Action action : regionAction.getActionList()) {
588          if (action.hasMutation()) {
589            numMutations++;
590          }
591          if (action.hasGet()) {
592            numGets++;
593          }
594          if (action.hasServiceCall()) {
595            numServiceCalls++;
596          }
597        }
598      }
599      responseInfo.put(MULTI_GETS, numGets);
600      responseInfo.put(MULTI_MUTATIONS, numMutations);
601      responseInfo.put(MULTI_SERVICE_CALLS, numServiceCalls);
602    }
603    final String tag =
604      (tooLarge && tooSlow) ? "TooLarge & TooSlow" : (tooSlow ? "TooSlow" : "TooLarge");
605    LOG.warn("(response" + tag + "): " + GSON.toJson(responseInfo));
606  }
607
608  private boolean isTooSlow(RpcCall call, int processingTime) {
609    long warnResponseTime = call.getParam() instanceof ClientProtos.ScanRequest
610      ? warnScanResponseTime
611      : this.warnResponseTime;
612    return (processingTime > warnResponseTime && warnResponseTime > -1);
613  }
614
615  private boolean isTooLarge(RpcCall call, long responseSize, long responseBlockSize) {
616    long warnResponseSize = call.getParam() instanceof ClientProtos.ScanRequest
617      ? warnScanResponseSize
618      : this.warnResponseSize;
619    return (warnResponseSize > -1
620      && (responseSize > warnResponseSize || responseBlockSize > warnResponseSize));
621  }
622
623  /**
624   * Truncate to number of chars decided by conf hbase.ipc.trace.log.max.length if TRACE is on else
625   * to 150 chars Refer to Jira HBASE-20826 and HBASE-20942
626   * @param strParam stringifiedParam to be truncated
627   * @return truncated trace log string
628   */
629  String truncateTraceLog(String strParam) {
630    if (LOG.isTraceEnabled()) {
631      int traceLogMaxLength = getConf().getInt(TRACE_LOG_MAX_LENGTH, DEFAULT_TRACE_LOG_MAX_LENGTH);
632      int truncatedLength =
633        strParam.length() < traceLogMaxLength ? strParam.length() : traceLogMaxLength;
634      String truncatedFlag = truncatedLength == strParam.length() ? "" : KEY_WORD_TRUNCATED;
635      return strParam.subSequence(0, truncatedLength) + truncatedFlag;
636    }
637    return strParam.subSequence(0, 150) + KEY_WORD_TRUNCATED;
638  }
639
640  /**
641   * Set the handler for calling out of RPC for error conditions.
642   * @param handler the handler implementation
643   */
644  @Override
645  public void setErrorHandler(HBaseRPCErrorHandler handler) {
646    this.errorHandler = handler;
647  }
648
649  @Override
650  public HBaseRPCErrorHandler getErrorHandler() {
651    return this.errorHandler;
652  }
653
654  /**
655   * Returns the metrics instance for reporting RPC call statistics
656   */
657  @Override
658  public MetricsHBaseServer getMetrics() {
659    return metrics;
660  }
661
662  @Override
663  public void addCallSize(final long diff) {
664    this.callQueueSizeInBytes.add(diff);
665  }
666
667  /**
668   * Authorize the incoming client connection.
669   * @param user       client user
670   * @param connection incoming connection
671   * @param addr       InetAddress of incoming connection
672   * @throws AuthorizationException when the client isn't authorized to talk the protocol
673   */
674  public synchronized void authorize(UserGroupInformation user, ConnectionHeader connection,
675    InetAddress addr) throws AuthorizationException {
676    if (authorize) {
677      Class<?> c = getServiceInterface(services, connection.getServiceName());
678      authManager.authorize(user, c, getConf(), addr);
679    }
680  }
681
682  /**
683   * When the read or write buffer size is larger than this limit, i/o will be done in chunks of
684   * this size. Most RPC requests and responses would be be smaller.
685   */
686  protected static final int NIO_BUFFER_LIMIT = 64 * 1024; // should not be more than 64KB.
687
688  /**
689   * This is a wrapper around
690   * {@link java.nio.channels.ReadableByteChannel#read(java.nio.ByteBuffer)}. If the amount of data
691   * is large, it writes to channel in smaller chunks. This is to avoid jdk from creating many
692   * direct buffers as the size of ByteBuffer increases. There should not be any performance
693   * degredation.
694   * @param channel writable byte channel to write on
695   * @param buffer  buffer to write
696   * @return number of bytes written
697   * @throws java.io.IOException e
698   * @see java.nio.channels.ReadableByteChannel#read(java.nio.ByteBuffer)
699   */
700  protected int channelRead(ReadableByteChannel channel, ByteBuffer buffer) throws IOException {
701
702    int count = (buffer.remaining() <= NIO_BUFFER_LIMIT)
703      ? channel.read(buffer)
704      : channelIO(channel, null, buffer);
705    if (count > 0) {
706      metrics.receivedBytes(count);
707    }
708    return count;
709  }
710
711  /**
712   * Helper for {@link #channelRead(java.nio.channels.ReadableByteChannel, java.nio.ByteBuffer)}.
713   * Only one of readCh or writeCh should be non-null.
714   * @param readCh  read channel
715   * @param writeCh write channel
716   * @param buf     buffer to read or write into/out of
717   * @return bytes written
718   * @throws java.io.IOException e
719   * @see #channelRead(java.nio.channels.ReadableByteChannel, java.nio.ByteBuffer)
720   */
721  private static int channelIO(ReadableByteChannel readCh, WritableByteChannel writeCh,
722    ByteBuffer buf) throws IOException {
723
724    int originalLimit = buf.limit();
725    int initialRemaining = buf.remaining();
726    int ret = 0;
727
728    while (buf.remaining() > 0) {
729      try {
730        int ioSize = Math.min(buf.remaining(), NIO_BUFFER_LIMIT);
731        buf.limit(buf.position() + ioSize);
732
733        ret = (readCh == null) ? writeCh.write(buf) : readCh.read(buf);
734
735        if (ret < ioSize) {
736          break;
737        }
738
739      } finally {
740        buf.limit(originalLimit);
741      }
742    }
743
744    int nBytes = initialRemaining - buf.remaining();
745    return (nBytes > 0) ? nBytes : ret;
746  }
747
748  /**
749   * Needed for features such as delayed calls. We need to be able to store the current call so that
750   * we can complete it later or ask questions of what is supported by the current ongoing call.
751   * @return An RpcCallContext backed by the currently ongoing call (gotten from a thread local)
752   */
753  public static Optional<RpcCall> getCurrentCall() {
754    return Optional.ofNullable(CurCall.get());
755  }
756
757  /**
758   * Just return the current rpc call if it is a {@link ServerCall} and also has {@link CellScanner}
759   * attached.
760   * <p/>
761   * Mainly used for reference counting as {@link CellScanner} may reference non heap memory.
762   */
763  public static Optional<ServerCall<?>> getCurrentServerCallWithCellScanner() {
764    return getCurrentCall().filter(c -> c instanceof ServerCall)
765      .filter(c -> c.getCellScanner() != null).map(c -> (ServerCall<?>) c);
766  }
767
768  public static boolean isInRpcCallContext() {
769    return CurCall.get() != null;
770  }
771
772  /**
773   * Used by {@link org.apache.hadoop.hbase.master.region.MasterRegion}, to avoid hit row lock
774   * timeout when updating master region in a rpc call. See HBASE-23895, HBASE-29251 and HBASE-29294
775   * for more details.
776   * @return the currently ongoing rpc call
777   */
778  public static Optional<RpcCall> unsetCurrentCall() {
779    Optional<RpcCall> rpcCall = getCurrentCall();
780    CurCall.set(null);
781    return rpcCall;
782  }
783
784  /**
785   * Used by {@link org.apache.hadoop.hbase.master.region.MasterRegion}. Set the rpc call back after
786   * mutate region.
787   */
788  public static void setCurrentCall(RpcCall rpcCall) {
789    CurCall.set(rpcCall);
790  }
791
792  /**
793   * Returns the user credentials associated with the current RPC request or not present if no
794   * credentials were provided.
795   * @return A User
796   */
797  public static Optional<User> getRequestUser() {
798    Optional<RpcCall> ctx = getCurrentCall();
799    return ctx.isPresent() ? ctx.get().getRequestUser() : Optional.empty();
800  }
801
802  /**
803   * The number of open RPC conections
804   * @return the number of open rpc connections
805   */
806  abstract public int getNumOpenConnections();
807
808  /**
809   * Returns the username for any user associated with the current RPC request or not present if no
810   * user is set.
811   */
812  public static Optional<String> getRequestUserName() {
813    return getRequestUser().map(User::getShortName);
814  }
815
816  /**
817   * Returns the address of the remote client associated with the current RPC request or not present
818   * if no address is set.
819   */
820  public static Optional<InetAddress> getRemoteAddress() {
821    return getCurrentCall().map(RpcCall::getRemoteAddress);
822  }
823
824  /**
825   * @param serviceName Some arbitrary string that represents a 'service'.
826   * @param services    Available service instances
827   * @return Matching BlockingServiceAndInterface pair
828   */
829  protected static BlockingServiceAndInterface getServiceAndInterface(
830    final List<BlockingServiceAndInterface> services, final String serviceName) {
831    for (BlockingServiceAndInterface bs : services) {
832      if (bs.getBlockingService().getDescriptorForType().getName().equals(serviceName)) {
833        return bs;
834      }
835    }
836    return null;
837  }
838
839  /**
840   * @param serviceName Some arbitrary string that represents a 'service'.
841   * @param services    Available services and their service interfaces.
842   * @return Service interface class for <code>serviceName</code>
843   */
844  protected static Class<?> getServiceInterface(final List<BlockingServiceAndInterface> services,
845    final String serviceName) {
846    BlockingServiceAndInterface bsasi = getServiceAndInterface(services, serviceName);
847    return bsasi == null ? null : bsasi.getServiceInterface();
848  }
849
850  /**
851   * @param serviceName Some arbitrary string that represents a 'service'.
852   * @param services    Available services and their service interfaces.
853   * @return BlockingService that goes with the passed <code>serviceName</code>
854   */
855  protected static BlockingService getService(final List<BlockingServiceAndInterface> services,
856    final String serviceName) {
857    BlockingServiceAndInterface bsasi = getServiceAndInterface(services, serviceName);
858    return bsasi == null ? null : bsasi.getBlockingService();
859  }
860
861  protected static MonitoredRPCHandler getStatus() {
862    // It is ugly the way we park status up in RpcServer. Let it be for now. TODO.
863    MonitoredRPCHandler status = RpcServer.MONITORED_RPC.get();
864    if (status != null) {
865      return status;
866    }
867    status = TaskMonitor.get().createRPCStatus(Thread.currentThread().getName());
868    status.pause("Waiting for a call");
869    RpcServer.MONITORED_RPC.set(status);
870    return status;
871  }
872
873  /**
874   * Returns the remote side ip address when invoked inside an RPC Returns null incase of an error.
875   */
876  public static InetAddress getRemoteIp() {
877    RpcCall call = CurCall.get();
878    if (call != null) {
879      return call.getRemoteAddress();
880    }
881    return null;
882  }
883
884  @Override
885  public RpcScheduler getScheduler() {
886    return scheduler;
887  }
888
889  @Override
890  public ByteBuffAllocator getByteBuffAllocator() {
891    return this.bbAllocator;
892  }
893
894  @Override
895  public void setRsRpcServices(RSRpcServices rsRpcServices) {
896    this.rsRpcServices = rsRpcServices;
897  }
898
899  @Override
900  public void setNamedQueueRecorder(NamedQueueRecorder namedQueueRecorder) {
901    this.namedQueueRecorder = namedQueueRecorder;
902  }
903
904  protected boolean needAuthorization() {
905    return authorize;
906  }
907
908  @RestrictedApi(explanation = "Should only be called in tests", link = "",
909      allowedOnPath = ".*/src/test/.*")
910  public List<BlockingServiceAndInterface> getServices() {
911    return services;
912  }
913
914  private void initializeCoprocessorHost(Configuration conf) {
915    this.cpHost = new RpcCoprocessorHost(conf);
916  }
917
918  @Override
919  public RpcCoprocessorHost getRpcCoprocessorHost() {
920    return cpHost;
921  }
922}