/**
    * Licensed to the Apache Software Foundation (ASF) under one
    * or more contributor license agreements.  See the NOTICE file
    * distributed with this work for additional information
    * regarding copyright ownership.  The ASF licenses this file
    * to you under the Apache License, Version 2.0 (the
    * "License"); you may not use this file except in compliance
    * with the License.  You may obtain a copy of the License at
    *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  package org.apache.hadoop.hbase.client;
  
  import java.io.EOFException;
  import java.io.IOException;
  import java.io.SyncFailedException;
  import java.lang.reflect.UndeclaredThrowableException;
  import java.net.ConnectException;
  import java.net.SocketTimeoutException;
  import java.nio.channels.ClosedChannelException;
  import java.util.Map.Entry;
  import java.util.concurrent.ConcurrentHashMap;
  import java.util.concurrent.ConcurrentMap;
  import java.util.concurrent.TimeoutException;
  
  import org.apache.commons.lang.mutable.MutableBoolean;
  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.hbase.DoNotRetryIOException;
  import org.apache.hadoop.hbase.HConstants;
  import org.apache.hadoop.hbase.ServerName;
  import org.apache.hadoop.hbase.classification.InterfaceAudience;
  import org.apache.hadoop.hbase.exceptions.ConnectionClosingException;
  import org.apache.hadoop.hbase.exceptions.PreemptiveFastFailException;
  import org.apache.hadoop.hbase.ipc.FailedServerException;
  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  import org.apache.hadoop.ipc.RemoteException;
  
  /**
   * 
   * The concrete {@link RetryingCallerInterceptor} class that implements the preemptive fast fail
   * feature.
   * 
   * The motivation is as follows : 
   * In case where a large number of clients try and talk to a particular region server in hbase, if
   * the region server goes down due to network problems, we might end up in a scenario where
   * the clients would go into a state where they all start to retry.
   * This behavior will set off many of the threads in pretty much the same path and they all would be
   * sleeping giving rise to a state where the client either needs to create more threads to send new
   * requests to other hbase machines or block because the client cannot create anymore threads.
   * 
   * In most cases the clients might prefer to have a bound on the number of threads that are created
   * in order to send requests to hbase. This would mostly result in the client thread starvation.
   * 
   *  To circumvent this problem, the approach that is being taken here under is to let 1 of the many
   *  threads who are trying to contact the regionserver with connection problems and let the other
   *  threads get a {@link PreemptiveFastFailException} so that they can move on and take other
   *  requests.
   *  
   *  This would give the client more flexibility on the kind of action he would want to take in cases
   *  where the regionserver is down. He can either discard the requests and send a nack upstream
   *  faster or have an application level retry or buffer the requests up so as to send them down to
   *  hbase later.
   *
   */
  @InterfaceAudience.Private
  class PreemptiveFastFailInterceptor extends RetryingCallerInterceptor {
  
    private static final Log LOG = LogFactory
        .getLog(PreemptiveFastFailInterceptor.class);
  
    // amount of time to wait before we consider a server to be in fast fail
    // mode
    protected final long fastFailThresholdMilliSec;
  
    // Keeps track of failures when we cannot talk to a server. Helps in
    // fast failing clients if the server is down for a long time.
    protected final ConcurrentMap<ServerName, FailureInfo> repeatedFailuresMap =
        new ConcurrentHashMap<ServerName, FailureInfo>();
  
    // We populate repeatedFailuresMap every time there is a failure. So, to
    // keep it from growing unbounded, we garbage collect the failure information
    // every cleanupInterval.
    protected final long failureMapCleanupIntervalMilliSec;
  
    protected volatile long lastFailureMapCleanupTimeMilliSec;
  
    // clear failure Info. Used to clean out all entries.
    // A safety valve, in case the client does not exit the
    // fast fail mode for any reason.
    private long fastFailClearingTimeMilliSec;
  
   private final ThreadLocal<MutableBoolean> threadRetryingInFastFailMode =
       new ThreadLocal<MutableBoolean>();
 
   public PreemptiveFastFailInterceptor(Configuration conf) {
     this.fastFailThresholdMilliSec = conf.getLong(
         HConstants.HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS,
         HConstants.HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS_DEFAULT);
     this.failureMapCleanupIntervalMilliSec = conf.getLong(
         HConstants.HBASE_CLIENT_FAST_FAIL_CLEANUP_MS_DURATION_MS,
         HConstants.HBASE_CLIENT_FAST_FAIL_CLEANUP_DURATION_MS_DEFAULT);
     lastFailureMapCleanupTimeMilliSec = EnvironmentEdgeManager.currentTime();
   }
 
   public void intercept(FastFailInterceptorContext context)
       throws PreemptiveFastFailException {
     context.setFailureInfo(repeatedFailuresMap.get(context.getServer()));
     if (inFastFailMode(context.getServer()) && !currentThreadInFastFailMode()) {
       // In Fast-fail mode, all but one thread will fast fail. Check
       // if we are that one chosen thread.
       context.setRetryDespiteFastFailMode(shouldRetryInspiteOfFastFail(context
           .getFailureInfo()));
       if (!context.isRetryDespiteFastFailMode()) { // we don't have to retry
         LOG.debug("Throwing PFFE : " + context.getFailureInfo() + " tries : "
             + context.getTries());
         throw new PreemptiveFastFailException(
             context.getFailureInfo().numConsecutiveFailures.get(),
             context.getFailureInfo().timeOfFirstFailureMilliSec,
             context.getFailureInfo().timeOfLatestAttemptMilliSec, context.getServer());
       }
     }
     context.setDidTry(true);
   }
 
   public void handleFailure(FastFailInterceptorContext context,
       Throwable t) throws IOException {
     handleThrowable(t, context.getServer(),
         context.getCouldNotCommunicateWithServer());
   }
 
   public void updateFailureInfo(FastFailInterceptorContext context) {
     updateFailureInfoForServer(context.getServer(), context.getFailureInfo(),
         context.didTry(), context.getCouldNotCommunicateWithServer()
             .booleanValue(), context.isRetryDespiteFastFailMode());
   }
 
   /**
    * Handles failures encountered when communicating with a server.
    *
    * Updates the FailureInfo in repeatedFailuresMap to reflect the failure.
    * Throws RepeatedConnectException if the client is in Fast fail mode.
    *
    * @param serverName
    * @param t
    *          - the throwable to be handled.
    * @throws PreemptiveFastFailException
    */
   private void handleFailureToServer(ServerName serverName, Throwable t) {
     if (serverName == null || t == null) {
       return;
     }
     long currentTime = EnvironmentEdgeManager.currentTime();
     FailureInfo fInfo = repeatedFailuresMap.get(serverName);
     if (fInfo == null) {
       fInfo = new FailureInfo(currentTime);
       FailureInfo oldfInfo = repeatedFailuresMap.putIfAbsent(serverName, fInfo);
 
       if (oldfInfo != null) {
         fInfo = oldfInfo;
       }
     }
     fInfo.timeOfLatestAttemptMilliSec = currentTime;
     fInfo.numConsecutiveFailures.incrementAndGet();
   }
 
   public void handleThrowable(Throwable t1, ServerName serverName,
       MutableBoolean couldNotCommunicateWithServer) throws IOException {
     Throwable t2 = translateException(t1);
     boolean isLocalException = !(t2 instanceof RemoteException);
     if (isLocalException && isConnectionException(t2)) {
       couldNotCommunicateWithServer.setValue(true);
       handleFailureToServer(serverName, t2);
     }
   }
 
   private Throwable translateException(Throwable t) throws IOException {
     if (t instanceof NoSuchMethodError) {
       // We probably can't recover from this exception by retrying.
       LOG.error(t);
       throw (NoSuchMethodError) t;
     }
 
     if (t instanceof NullPointerException) {
       // The same here. This is probably a bug.
       LOG.error(t.getMessage(), t);
       throw (NullPointerException) t;
     }
 
     if (t instanceof UndeclaredThrowableException) {
       t = t.getCause();
     }
     if (t instanceof RemoteException) {
       t = ((RemoteException) t).unwrapRemoteException();
     }
     if (t instanceof DoNotRetryIOException) {
       throw (DoNotRetryIOException) t;
     }
     if (t instanceof NeedUnmanagedConnectionException) {
       throw new DoNotRetryIOException(t);
     }
     if (t instanceof Error) {
       throw (Error) t;
     }
     return t;
   }
 
   /**
    * Check if the exception is something that indicates that we cannot
    * contact/communicate with the server.
    *
    * @param e
    * @return true when exception indicates that the client wasn't able to make contact with server
    */
   private boolean isConnectionException(Throwable e) {
     if (e == null)
       return false;
     // This list covers most connectivity exceptions but not all.
     // For example, in SocketOutputStream a plain IOException is thrown
     // at times when the channel is closed.
     return (e instanceof SocketTimeoutException
         || e instanceof ConnectException || e instanceof ClosedChannelException
         || e instanceof SyncFailedException || e instanceof EOFException
         || e instanceof TimeoutException
         || e instanceof ConnectionClosingException || e instanceof FailedServerException);
   }
 
   /**
    * Occasionally cleans up unused information in repeatedFailuresMap.
    *
    * repeatedFailuresMap stores the failure information for all remote hosts
    * that had failures. In order to avoid these from growing indefinitely,
    * occassionallyCleanupFailureInformation() will clear these up once every
    * cleanupInterval ms.
    */
   protected void occasionallyCleanupFailureInformation() {
     long now = System.currentTimeMillis();
     if (!(now > lastFailureMapCleanupTimeMilliSec
         + failureMapCleanupIntervalMilliSec))
       return;
 
     // remove entries that haven't been attempted in a while
     // No synchronization needed. It is okay if multiple threads try to
     // remove the entry again and again from a concurrent hash map.
     StringBuilder sb = new StringBuilder();
     for (Entry<ServerName, FailureInfo> entry : repeatedFailuresMap.entrySet()) {
       if (now > entry.getValue().timeOfLatestAttemptMilliSec
           + failureMapCleanupIntervalMilliSec) { // no recent failures
         repeatedFailuresMap.remove(entry.getKey());
       } else if (now > entry.getValue().timeOfFirstFailureMilliSec
           + this.fastFailClearingTimeMilliSec) { // been failing for a long
                                                  // time
         LOG.error(entry.getKey()
             + " been failing for a long time. clearing out."
             + entry.getValue().toString());
         repeatedFailuresMap.remove(entry.getKey());
       } else {
         sb.append(entry.getKey().toString()).append(" failing ")
             .append(entry.getValue().toString()).append("\n");
       }
     }
     if (sb.length() > 0) {
       LOG.warn("Preemptive failure enabled for : " + sb.toString());
     }
     lastFailureMapCleanupTimeMilliSec = now;
   }
 
   /**
    * Checks to see if we are in the Fast fail mode for requests to the server.
    *
    * If a client is unable to contact a server for more than
    * fastFailThresholdMilliSec the client will get into fast fail mode.
    *
    * @param server
    * @return true if the client is in fast fail mode for the server.
    */
   private boolean inFastFailMode(ServerName server) {
     FailureInfo fInfo = repeatedFailuresMap.get(server);
     // if fInfo is null --> The server is considered good.
     // If the server is bad, wait long enough to believe that the server is
     // down.
     return (fInfo != null &&
         EnvironmentEdgeManager.currentTime() >
           (fInfo.timeOfFirstFailureMilliSec + this.fastFailThresholdMilliSec));
   }
 
   /**
    * Checks to see if the current thread is already in FastFail mode for *some*
    * server.
    *
    * @return true, if the thread is already in FF mode.
    */
   private boolean currentThreadInFastFailMode() {
     return (this.threadRetryingInFastFailMode.get() != null && (this.threadRetryingInFastFailMode
         .get().booleanValue() == true));
   }
 
   /**
    * Check to see if the client should try to connnect to the server, inspite of
    * knowing that it is in the fast fail mode.
    *
    * The idea here is that we want just one client thread to be actively trying
    * to reconnect, while all the other threads trying to reach the server will
    * short circuit.
    *
    * @param fInfo
    * @return true if the client should try to connect to the server.
    */
   protected boolean shouldRetryInspiteOfFastFail(FailureInfo fInfo) {
     // We believe that the server is down, But, we want to have just one
     // client
     // actively trying to connect. If we are the chosen one, we will retry
     // and not throw an exception.
     if (fInfo != null
         && fInfo.exclusivelyRetringInspiteOfFastFail.compareAndSet(false, true)) {
       MutableBoolean threadAlreadyInFF = this.threadRetryingInFastFailMode
           .get();
       if (threadAlreadyInFF == null) {
         threadAlreadyInFF = new MutableBoolean();
         this.threadRetryingInFastFailMode.set(threadAlreadyInFF);
       }
       threadAlreadyInFF.setValue(true);
       return true;
     } else {
       return false;
     }
   }
 
   /**
    *
    * This function updates the Failure info for a particular server after the
    * attempt to 
    *
    * @param server
    * @param fInfo
    * @param couldNotCommunicate
    * @param retryDespiteFastFailMode
    */
   private void updateFailureInfoForServer(ServerName server,
       FailureInfo fInfo, boolean didTry, boolean couldNotCommunicate,
       boolean retryDespiteFastFailMode) {
     if (server == null || fInfo == null || didTry == false)
       return;
 
     // If we were able to connect to the server, reset the failure
     // information.
     if (couldNotCommunicate == false) {
       LOG.info("Clearing out PFFE for server " + server.getServerName());
       repeatedFailuresMap.remove(server);
     } else {
       // update time of last attempt
       long currentTime = System.currentTimeMillis();
       fInfo.timeOfLatestAttemptMilliSec = currentTime;
 
       // Release the lock if we were retrying inspite of FastFail
       if (retryDespiteFastFailMode) {
         fInfo.exclusivelyRetringInspiteOfFastFail.set(false);
         threadRetryingInFastFailMode.get().setValue(false);
       }
     }
 
     occasionallyCleanupFailureInformation();
   }
 
   @Override
   public void intercept(RetryingCallerInterceptorContext context)
       throws PreemptiveFastFailException {
     if (context instanceof FastFailInterceptorContext) {
       intercept((FastFailInterceptorContext) context);
     }
   }
 
   @Override
   public void handleFailure(RetryingCallerInterceptorContext context,
       Throwable t) throws IOException {
     if (context instanceof FastFailInterceptorContext) {
       handleFailure((FastFailInterceptorContext) context, t);
     }
   }
 
   @Override
   public void updateFailureInfo(RetryingCallerInterceptorContext context) {
     if (context instanceof FastFailInterceptorContext) {
       updateFailureInfo((FastFailInterceptorContext) context);
     }
   }
 
   @Override
   public RetryingCallerInterceptorContext createEmptyContext() {
     return new FastFailInterceptorContext();
   }
 
   protected boolean isServerInFailureMap(ServerName serverName) {
     return this.repeatedFailuresMap.containsKey(serverName);
   }
 
   @Override
   public String toString() {
     return "PreemptiveFastFailInterceptor";
   }
 }