/**
    * Licensed to the Apache Software Foundation (ASF) under one
    * or more contributor license agreements.  See the NOTICE file
    * distributed with this work for additional information
    * regarding copyright ownership.  The ASF licenses this file
    * to you under the Apache License, Version 2.0 (the
    * "License"); you may not use this file except in compliance
    * with the License.  You may obtain a copy of the License at
    *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  package org.apache.hadoop.hbase.procedure;
  
  import java.io.IOException;
  import java.util.ArrayList;
  import java.util.HashMap;
  import java.util.List;
  import java.util.concurrent.Callable;
  import java.util.concurrent.CountDownLatch;
  import java.util.concurrent.TimeUnit;
  
  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.hbase.classification.InterfaceAudience;
  import org.apache.hadoop.hbase.errorhandling.ForeignException;
  import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
  import org.apache.hadoop.hbase.errorhandling.ForeignExceptionListener;
  import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
  import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
  
  import com.google.common.collect.Lists;
  
  /**
   * A globally-barriered distributed procedure.  This class encapsulates state and methods for
   * tracking and managing a distributed procedure, as well as aborting if any member encounters
   * a problem or if a cancellation is requested.
   * <p>
   * All procedures first attempt to reach a barrier point with the {@link #sendGlobalBarrierStart()}
   * method.  The procedure contacts all members and waits for all subprocedures to execute
   * {@link Subprocedure#acquireBarrier} to acquire its local piece of the global barrier and then
   * send acquisition info back to the coordinator.  If all acquisitions at subprocedures succeed,
   * the coordinator then will call {@link #sendGlobalBarrierReached()}.  This notifies members to
   * execute the {@link Subprocedure#insideBarrier()} method.  The procedure is blocked until all
   * {@link Subprocedure#insideBarrier} executions complete at the members.  When
   * {@link Subprocedure#insideBarrier} completes at each member, the member sends notification to
   * the coordinator.  Once all members complete, the coordinator calls
   * {@link #sendGlobalBarrierComplete()}.
   * <p>
   * If errors are encountered remotely, they are forwarded to the coordinator, and
   * {@link Subprocedure#cleanup(Exception)} is called.
   * <p>
   * Each Procedure and each Subprocedure enforces a time limit on the execution time. If the time
   * limit expires before the procedure completes the {@link TimeoutExceptionInjector} will trigger
   * an {@link ForeignException} to abort the procedure.  This is particularly useful for situations
   * when running a distributed {@link Subprocedure} so participants can avoid blocking for extreme
   * amounts of time if one of the participants fails or takes a really long time (e.g. GC pause).
   * <p>
   * Users should generally not directly create or subclass instances of this.  They are created
   * for them implicitly via {@link ProcedureCoordinator#startProcedure(ForeignExceptionDispatcher,
   * String, byte[], List)}}
   */
  @InterfaceAudience.Private
  public class Procedure implements Callable<Void>, ForeignExceptionListener {
    private static final Log LOG = LogFactory.getLog(Procedure.class);
  
    //
    // Arguments and naming
    //
  
    // Name of the procedure
    final private String procName;
    // Arguments for this procedure execution
    final private byte[] args;
  
    //
    // Execution State
    //
    /** latch for waiting until all members have acquire in barrier state */
    final CountDownLatch acquiredBarrierLatch;
    /** latch for waiting until all members have executed and released their in barrier state */
    final CountDownLatch releasedBarrierLatch;
    /** latch for waiting until a procedure has completed */
    final CountDownLatch completedLatch;
    /** monitor to check for errors */
    private final ForeignExceptionDispatcher monitor;
  
    //
    // Execution Timeout Handling.
    //
  
    /** frequency to check for errors (ms) */
    protected final long wakeFrequency;
    protected final TimeoutExceptionInjector timeoutInjector;
 
   //
   // Members' and Coordinator's state
   //
 
   /** lock to prevent nodes from acquiring and then releasing before we can track them */
   private Object joinBarrierLock = new Object();
   private final List<String> acquiringMembers;
   private final List<String> inBarrierMembers;
   private final HashMap<String, byte[]> dataFromFinishedMembers;
   private ProcedureCoordinator coord;
 
   /**
    * Creates a procedure. (FOR TESTING)
    *
    * {@link Procedure} state to be run by a {@link ProcedureCoordinator}.
    * @param coord coordinator to call back to for general errors (e.g.
    *          {@link ProcedureCoordinator#rpcConnectionFailure(String, IOException)}).
    * @param monitor error monitor to check for external errors
    * @param wakeFreq frequency to check for errors while waiting
    * @param timeout amount of time to allow the procedure to run before cancelling
    * @param procName name of the procedure instance
    * @param args argument data associated with the procedure instance
    * @param expectedMembers names of the expected members
    */
   public Procedure(ProcedureCoordinator coord, ForeignExceptionDispatcher monitor, long wakeFreq,
       long timeout, String procName, byte[] args, List<String> expectedMembers) {
     this.coord = coord;
     this.acquiringMembers = new ArrayList<String>(expectedMembers);
     this.inBarrierMembers = new ArrayList<String>(acquiringMembers.size());
     this.dataFromFinishedMembers = new HashMap<String, byte[]>();
     this.procName = procName;
     this.args = args;
     this.monitor = monitor;
     this.wakeFrequency = wakeFreq;
 
     int count = expectedMembers.size();
     this.acquiredBarrierLatch = new CountDownLatch(count);
     this.releasedBarrierLatch = new CountDownLatch(count);
     this.completedLatch = new CountDownLatch(1);
     this.timeoutInjector = new TimeoutExceptionInjector(monitor, timeout);
   }
 
   /**
    * Create a procedure.
    *
    * Users should generally not directly create instances of this.  They are created them
    * implicitly via {@link ProcedureCoordinator#createProcedure(ForeignExceptionDispatcher,
    * String, byte[], List)}}
    *
    * @param coord coordinator to call back to for general errors (e.g.
    *          {@link ProcedureCoordinator#rpcConnectionFailure(String, IOException)}).
    * @param wakeFreq frequency to check for errors while waiting
    * @param timeout amount of time to allow the procedure to run before cancelling
    * @param procName name of the procedure instance
    * @param args argument data associated with the procedure instance
    * @param expectedMembers names of the expected members
    */
   public Procedure(ProcedureCoordinator coord, long wakeFreq, long timeout,
       String procName, byte[] args, List<String> expectedMembers) {
     this(coord, new ForeignExceptionDispatcher(), wakeFreq, timeout, procName, args,
         expectedMembers);
   }
 
   public String getName() {
     return procName;
   }
 
   /**
    * @return String of the procedure members both trying to enter the barrier and already in barrier
    */
   public String getStatus() {
     String waiting, done;
     synchronized (joinBarrierLock) {
       waiting = acquiringMembers.toString();
       done = inBarrierMembers.toString();
     }
     return "Procedure " + procName + " { waiting=" + waiting + " done="+ done + " }";
   }
 
   /**
    * Get the ForeignExceptionDispatcher
    * @return the Procedure's monitor.
    */
   public ForeignExceptionDispatcher getErrorMonitor() {
     return monitor;
   }
 
   /**
    * This call is the main execution thread of the barriered procedure.  It sends messages and
    * essentially blocks until all procedure members acquire or later complete but periodically
    * checks for foreign exceptions.
    */
   @Override
   @SuppressWarnings("finally")
   final public Void call() {
     LOG.info("Starting procedure '" + procName + "'");
     // start the timer
     timeoutInjector.start();
 
     // run the procedure
     try {
       // start by checking for error first
       monitor.rethrowException();
       LOG.debug("Procedure '" + procName + "' starting 'acquire'");
       sendGlobalBarrierStart();
 
       // wait for all the members to report acquisition
       LOG.debug("Waiting for all members to 'acquire'");
       waitForLatch(acquiredBarrierLatch, monitor, wakeFrequency, "acquired");
       monitor.rethrowException();
 
       LOG.debug("Procedure '" + procName + "' starting 'in-barrier' execution.");
       sendGlobalBarrierReached();
 
       // wait for all members to report barrier release
       LOG.debug("Waiting for all members to 'release'");
       waitForLatch(releasedBarrierLatch, monitor, wakeFrequency, "released");
 
       // make sure we didn't get an error during in barrier execution and release
       monitor.rethrowException();
       LOG.info("Procedure '" + procName + "' execution completed");
     } catch (Exception e) {
       if (e instanceof InterruptedException) {
         Thread.currentThread().interrupt();
       }
       String msg = "Procedure '" + procName +"' execution failed!";
       LOG.error(msg, e);
       receive(new ForeignException(getName(), e));
     } finally {
       LOG.debug("Running finish phase.");
       sendGlobalBarrierComplete();
       completedLatch.countDown();
 
       // tell the timer we are done, if we get here successfully
       timeoutInjector.complete();
       return null;
     }
   }
 
   /**
    * Sends a message to Members to create a new {@link Subprocedure} for this Procedure and execute
    * the {@link Subprocedure#acquireBarrier} step.
    * @throws ForeignException
    */
   public void sendGlobalBarrierStart() throws ForeignException {
     // start the procedure
     LOG.debug("Starting procedure '" + procName + "', kicking off acquire phase on members.");
     try {
       // send procedure barrier start to specified list of members. cloning the list to avoid
       // concurrent modification from the controller setting the prepared nodes
       coord.getRpcs().sendGlobalBarrierAcquire(this, args, Lists.newArrayList(this.acquiringMembers));
     } catch (IOException e) {
       coord.rpcConnectionFailure("Can't reach controller.", e);
     } catch (IllegalArgumentException e) {
       throw new ForeignException(getName(), e);
     }
   }
 
   /**
    * Sends a message to all members that the global barrier condition has been satisfied.  This
    * should only be executed after all members have completed its
    * {@link Subprocedure#acquireBarrier()} call successfully.  This triggers the member
    * {@link Subprocedure#insideBarrier} method.
    * @throws ForeignException
    */
   public void sendGlobalBarrierReached() throws ForeignException {
     try {
       // trigger to have member run {@link Subprocedure#insideBarrier}
       coord.getRpcs().sendGlobalBarrierReached(this, Lists.newArrayList(inBarrierMembers));
     } catch (IOException e) {
       coord.rpcConnectionFailure("Can't reach controller.", e);
     }
   }
 
   /**
    * Sends a message to members that all {@link Subprocedure#insideBarrier} calls have completed.
    * After this executes, the coordinator can assume that any state resources about this barrier
    * procedure state has been released.
    */
   public void sendGlobalBarrierComplete() {
     LOG.debug("Finished coordinator procedure - removing self from list of running procedures");
     try {
       coord.getRpcs().resetMembers(this);
     } catch (IOException e) {
       coord.rpcConnectionFailure("Failed to reset procedure:" + procName, e);
     }
   }
 
   //
   // Call backs from other external processes.
   //
 
   /**
    * Call back triggered by an individual member upon successful local barrier acquisition
    * @param member
    */
   public void barrierAcquiredByMember(String member) {
     LOG.debug("member: '" + member + "' joining acquired barrier for procedure '" + procName
         + "' on coordinator");
     if (this.acquiringMembers.contains(member)) {
       synchronized (joinBarrierLock) {
         if (this.acquiringMembers.remove(member)) {
           this.inBarrierMembers.add(member);
           acquiredBarrierLatch.countDown();
         }
       }
       LOG.debug("Waiting on: " + acquiredBarrierLatch + " remaining members to acquire global barrier");
     } else {
       LOG.warn("Member " + member + " joined barrier, but we weren't waiting on it to join." +
           " Continuing on.");
     }
   }
 
   /**
    * Call back triggered by a individual member upon successful local in-barrier execution and
    * release
    * @param member
    * @param dataFromMember
    */
   public void barrierReleasedByMember(String member, byte[] dataFromMember) {
     boolean removed = false;
     synchronized (joinBarrierLock) {
       removed = this.inBarrierMembers.remove(member);
       if (removed) {
         releasedBarrierLatch.countDown();
       }
     }
     if (removed) {
       LOG.debug("Member: '" + member + "' released barrier for procedure'" + procName
           + "', counting down latch.  Waiting for " + releasedBarrierLatch.getCount()
           + " more");
     } else {
       LOG.warn("Member: '" + member + "' released barrier for procedure'" + procName
           + "', but we weren't waiting on it to release!");
     }
     dataFromFinishedMembers.put(member, dataFromMember);
   }
 
   /**
    * Waits until the entire procedure has globally completed, or has been aborted.  If an
    * exception is thrown the procedure may or not have run cleanup to trigger the completion latch
    * yet.
    * @throws ForeignException
    * @throws InterruptedException
    */
   public void waitForCompleted() throws ForeignException, InterruptedException {
     waitForLatch(completedLatch, monitor, wakeFrequency, procName + " completed");
   }
 
   /**
    * Waits until the entire procedure has globally completed, or has been aborted.  If an
    * exception is thrown the procedure may or not have run cleanup to trigger the completion latch
    * yet.
    * @return data returned from procedure members upon successfully completing subprocedure.
    * @throws ForeignException
    * @throws InterruptedException
    */
   public HashMap<String, byte[]> waitForCompletedWithRet() throws ForeignException, InterruptedException {
     waitForCompleted();
     return dataFromFinishedMembers;
   }
 
   /**
    * Check if the entire procedure has globally completed, or has been aborted.
    * @throws ForeignException
    */
   public boolean isCompleted() throws ForeignException {
     // Rethrow exception if any
     monitor.rethrowException();
     return (completedLatch.getCount() == 0);
   }
 
   /**
    * A callback that handles incoming ForeignExceptions.
    */
   @Override
   public void receive(ForeignException e) {
     monitor.receive(e);
   }
 
   /**
    * Wait for latch to count to zero, ignoring any spurious wake-ups, but waking periodically to
    * check for errors
    * @param latch latch to wait on
    * @param monitor monitor to check for errors while waiting
    * @param wakeFrequency frequency to wake up and check for errors (in
    *          {@link TimeUnit#MILLISECONDS})
    * @param latchDescription description of the latch, for logging
    * @throws ForeignException type of error the monitor can throw, if the task fails
    * @throws InterruptedException if we are interrupted while waiting on latch
    */
   public static void waitForLatch(CountDownLatch latch, ForeignExceptionSnare monitor,
       long wakeFrequency, String latchDescription) throws ForeignException,
       InterruptedException {
     boolean released = false;
     while (!released) {
       if (monitor != null) {
         monitor.rethrowException();
       }
       /*
       ForeignExceptionDispatcher.LOG.debug("Waiting for '" + latchDescription + "' latch. (sleep:"
           + wakeFrequency + " ms)"); */
       released = latch.await(wakeFrequency, TimeUnit.MILLISECONDS);
     }
     // check error again in case an error raised during last wait
     if (monitor != null) {
       monitor.rethrowException();
     }
   }
 }