001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hbase.coprocessor;
020
021import java.io.IOException;
022import java.util.ArrayList;
023import java.util.Collections;
024import java.util.Comparator;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Optional;
028import java.util.Set;
029import java.util.TreeSet;
030import java.util.UUID;
031import java.util.concurrent.ConcurrentSkipListSet;
032import java.util.concurrent.atomic.AtomicInteger;
033import java.util.function.Function;
034
035import org.apache.yetus.audience.InterfaceAudience;
036import org.slf4j.Logger;
037import org.slf4j.LoggerFactory;
038import org.apache.hadoop.conf.Configuration;
039import org.apache.hadoop.fs.Path;
040import org.apache.hadoop.hbase.Abortable;
041import org.apache.hadoop.hbase.Coprocessor;
042import org.apache.hadoop.hbase.CoprocessorEnvironment;
043import org.apache.hadoop.hbase.DoNotRetryIOException;
044import org.apache.hadoop.hbase.HConstants;
045import org.apache.hadoop.hbase.ipc.RpcServer;
046import org.apache.hadoop.hbase.security.User;
047import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
048import org.apache.hadoop.hbase.util.SortedList;
049
050/**
051 * Provides the common setup framework and runtime services for coprocessor
052 * invocation from HBase services.
053 * @param <C> type of specific coprocessor this host will handle
054 * @param <E> type of specific coprocessor environment this host requires.
055 * provides
056 */
057@InterfaceAudience.Private
058public abstract class CoprocessorHost<C extends Coprocessor, E extends CoprocessorEnvironment<C>> {
059  public static final String REGION_COPROCESSOR_CONF_KEY =
060      "hbase.coprocessor.region.classes";
061  public static final String REGIONSERVER_COPROCESSOR_CONF_KEY =
062      "hbase.coprocessor.regionserver.classes";
063  public static final String USER_REGION_COPROCESSOR_CONF_KEY =
064      "hbase.coprocessor.user.region.classes";
065  public static final String MASTER_COPROCESSOR_CONF_KEY =
066      "hbase.coprocessor.master.classes";
067  public static final String WAL_COPROCESSOR_CONF_KEY =
068    "hbase.coprocessor.wal.classes";
069  public static final String ABORT_ON_ERROR_KEY = "hbase.coprocessor.abortonerror";
070  public static final boolean DEFAULT_ABORT_ON_ERROR = true;
071  public static final String COPROCESSORS_ENABLED_CONF_KEY = "hbase.coprocessor.enabled";
072  public static final boolean DEFAULT_COPROCESSORS_ENABLED = true;
073  public static final String USER_COPROCESSORS_ENABLED_CONF_KEY =
074    "hbase.coprocessor.user.enabled";
075  public static final boolean DEFAULT_USER_COPROCESSORS_ENABLED = true;
076  public static final String SKIP_LOAD_DUPLICATE_TABLE_COPROCESSOR =
077      "hbase.skip.load.duplicate.table.coprocessor";
078  public static final boolean DEFAULT_SKIP_LOAD_DUPLICATE_TABLE_COPROCESSOR = false;
079
080  private static final Logger LOG = LoggerFactory.getLogger(CoprocessorHost.class);
081  protected Abortable abortable;
082  /** Ordered set of loaded coprocessors with lock */
083  protected final SortedList<E> coprocEnvironments =
084      new SortedList<>(new EnvironmentPriorityComparator());
085  protected Configuration conf;
086  // unique file prefix to use for local copies of jars when classloading
087  protected String pathPrefix;
088  protected AtomicInteger loadSequence = new AtomicInteger();
089
090  public CoprocessorHost(Abortable abortable) {
091    this.abortable = abortable;
092    this.pathPrefix = UUID.randomUUID().toString();
093  }
094
095  /**
096   * Not to be confused with the per-object _coprocessors_ (above),
097   * coprocessorNames is static and stores the set of all coprocessors ever
098   * loaded by any thread in this JVM. It is strictly additive: coprocessors are
099   * added to coprocessorNames, by checkAndLoadInstance() but are never removed, since
100   * the intention is to preserve a history of all loaded coprocessors for
101   * diagnosis in case of server crash (HBASE-4014).
102   */
103  private static Set<String> coprocessorNames =
104      Collections.synchronizedSet(new HashSet<String>());
105
106  public static Set<String> getLoadedCoprocessors() {
107    synchronized (coprocessorNames) {
108      return new HashSet(coprocessorNames);
109    }
110  }
111
112  /**
113   * Used to create a parameter to the HServerLoad constructor so that
114   * HServerLoad can provide information about the coprocessors loaded by this
115   * regionserver.
116   * (HBASE-4070: Improve region server metrics to report loaded coprocessors
117   * to master).
118   */
119  public Set<String> getCoprocessors() {
120    Set<String> returnValue = new TreeSet<>();
121    for (E e: coprocEnvironments) {
122      returnValue.add(e.getInstance().getClass().getSimpleName());
123    }
124    return returnValue;
125  }
126
127  /**
128   * Load system coprocessors once only. Read the class names from configuration.
129   * Called by constructor.
130   */
131  protected void loadSystemCoprocessors(Configuration conf, String confKey) {
132    boolean coprocessorsEnabled = conf.getBoolean(COPROCESSORS_ENABLED_CONF_KEY,
133      DEFAULT_COPROCESSORS_ENABLED);
134    if (!coprocessorsEnabled) {
135      return;
136    }
137
138    Class<?> implClass;
139
140    // load default coprocessors from configure file
141    String[] defaultCPClasses = conf.getStrings(confKey);
142    if (defaultCPClasses == null || defaultCPClasses.length == 0)
143      return;
144
145    int currentSystemPriority = Coprocessor.PRIORITY_SYSTEM;
146    for (String className : defaultCPClasses) {
147      String[] classNameAndPriority = className.split("\\|");
148      boolean hasPriorityOverride = false;
149      className = classNameAndPriority[0];
150      int overridePriority = Coprocessor.PRIORITY_SYSTEM;
151      if (classNameAndPriority.length > 1){
152        overridePriority = Integer.parseInt(classNameAndPriority[1]);
153        hasPriorityOverride = true;
154      }
155      className = className.trim();
156      if (findCoprocessor(className) != null) {
157        // If already loaded will just continue
158        LOG.warn("Attempted duplicate loading of " + className + "; skipped");
159        continue;
160      }
161      ClassLoader cl = this.getClass().getClassLoader();
162      Thread.currentThread().setContextClassLoader(cl);
163      try {
164        implClass = cl.loadClass(className);
165        int coprocPriority = hasPriorityOverride ? overridePriority : currentSystemPriority;
166        // Add coprocessors as we go to guard against case where a coprocessor is specified twice
167        // in the configuration
168        E env = checkAndLoadInstance(implClass, coprocPriority, conf);
169        if (env != null) {
170          this.coprocEnvironments.add(env);
171          LOG.info("System coprocessor {} loaded, priority={}.", className, coprocPriority);
172          if (!hasPriorityOverride) {
173            ++currentSystemPriority;
174          }
175        }
176      } catch (Throwable t) {
177        // We always abort if system coprocessors cannot be loaded
178        abortServer(className, t);
179      }
180    }
181  }
182
183  /**
184   * Load a coprocessor implementation into the host
185   * @param path path to implementation jar
186   * @param className the main class name
187   * @param priority chaining priority
188   * @param conf configuration for coprocessor
189   * @throws java.io.IOException Exception
190   */
191  public E load(Path path, String className, int priority,
192      Configuration conf) throws IOException {
193    String[] includedClassPrefixes = null;
194    if (conf.get(HConstants.CP_HTD_ATTR_INCLUSION_KEY) != null){
195      String prefixes = conf.get(HConstants.CP_HTD_ATTR_INCLUSION_KEY);
196      includedClassPrefixes = prefixes.split(";");
197    }
198    return load(path, className, priority, conf, includedClassPrefixes);
199  }
200
201  /**
202   * Load a coprocessor implementation into the host
203   * @param path path to implementation jar
204   * @param className the main class name
205   * @param priority chaining priority
206   * @param conf configuration for coprocessor
207   * @param includedClassPrefixes class name prefixes to include
208   * @throws java.io.IOException Exception
209   */
210  public E load(Path path, String className, int priority,
211      Configuration conf, String[] includedClassPrefixes) throws IOException {
212    Class<?> implClass;
213    LOG.debug("Loading coprocessor class " + className + " with path " +
214        path + " and priority " + priority);
215
216    boolean skipLoadDuplicateCoprocessor = conf.getBoolean(SKIP_LOAD_DUPLICATE_TABLE_COPROCESSOR,
217      DEFAULT_SKIP_LOAD_DUPLICATE_TABLE_COPROCESSOR);
218    if (skipLoadDuplicateCoprocessor && findCoprocessor(className) != null) {
219      // If already loaded will just continue
220      LOG.warn("Attempted duplicate loading of {}; skipped", className);
221      return null;
222    }
223
224    ClassLoader cl = null;
225    if (path == null) {
226      try {
227        implClass = getClass().getClassLoader().loadClass(className);
228      } catch (ClassNotFoundException e) {
229        throw new IOException("No jar path specified for " + className);
230      }
231    } else {
232      cl = CoprocessorClassLoader.getClassLoader(
233        path, getClass().getClassLoader(), pathPrefix, conf);
234      try {
235        implClass = ((CoprocessorClassLoader)cl).loadClass(className, includedClassPrefixes);
236      } catch (ClassNotFoundException e) {
237        throw new IOException("Cannot load external coprocessor class " + className, e);
238      }
239    }
240
241    //load custom code for coprocessor
242    Thread currentThread = Thread.currentThread();
243    ClassLoader hostClassLoader = currentThread.getContextClassLoader();
244    try{
245      // switch temporarily to the thread classloader for custom CP
246      currentThread.setContextClassLoader(cl);
247      E cpInstance = checkAndLoadInstance(implClass, priority, conf);
248      return cpInstance;
249    } finally {
250      // restore the fresh (host) classloader
251      currentThread.setContextClassLoader(hostClassLoader);
252    }
253  }
254
255  public void load(Class<? extends C> implClass, int priority, Configuration conf)
256      throws IOException {
257    E env = checkAndLoadInstance(implClass, priority, conf);
258    coprocEnvironments.add(env);
259  }
260
261  /**
262   * @param implClass Implementation class
263   * @param priority priority
264   * @param conf configuration
265   * @throws java.io.IOException Exception
266   */
267  public E checkAndLoadInstance(Class<?> implClass, int priority, Configuration conf)
268      throws IOException {
269    // create the instance
270    C impl;
271    try {
272      impl = checkAndGetInstance(implClass);
273      if (impl == null) {
274        LOG.error("Cannot load coprocessor " + implClass.getSimpleName());
275        return null;
276      }
277    } catch (InstantiationException|IllegalAccessException e) {
278      throw new IOException(e);
279    }
280    // create the environment
281    E env = createEnvironment(impl, priority, loadSequence.incrementAndGet(), conf);
282    assert env instanceof BaseEnvironment;
283    ((BaseEnvironment<C>) env).startup();
284    // HBASE-4014: maintain list of loaded coprocessors for later crash analysis
285    // if server (master or regionserver) aborts.
286    coprocessorNames.add(implClass.getName());
287    return env;
288  }
289
290  /**
291   * Called when a new Coprocessor class is loaded
292   */
293  public abstract E createEnvironment(C instance, int priority, int sequence, Configuration conf);
294
295  /**
296   * Called when a new Coprocessor class needs to be loaded. Checks if type of the given class
297   * is what the corresponding host implementation expects. If it is of correct type, returns an
298   * instance of the coprocessor to be loaded. If not, returns null.
299   * If an exception occurs when trying to create instance of a coprocessor, it's passed up and
300   * eventually results into server aborting.
301   */
302  public abstract C checkAndGetInstance(Class<?> implClass)
303      throws InstantiationException, IllegalAccessException;
304
305  public void shutdown(E e) {
306    assert e instanceof BaseEnvironment;
307    if (LOG.isDebugEnabled()) {
308      LOG.debug("Stop coprocessor " + e.getInstance().getClass().getName());
309    }
310    ((BaseEnvironment<C>) e).shutdown();
311  }
312
313  /**
314   * Find coprocessors by full class name or simple name.
315   */
316  public C findCoprocessor(String className) {
317    for (E env: coprocEnvironments) {
318      if (env.getInstance().getClass().getName().equals(className) ||
319          env.getInstance().getClass().getSimpleName().equals(className)) {
320        return env.getInstance();
321      }
322    }
323    return null;
324  }
325
326  public <T extends C> T findCoprocessor(Class<T> cls) {
327    for (E env: coprocEnvironments) {
328      if (cls.isAssignableFrom(env.getInstance().getClass())) {
329        return (T) env.getInstance();
330      }
331    }
332    return null;
333  }
334
335  /**
336   * Find list of coprocessors that extend/implement the given class/interface
337   * @param cls the class/interface to look for
338   * @return the list of coprocessors, or null if not found
339   */
340  public <T extends C> List<T> findCoprocessors(Class<T> cls) {
341    ArrayList<T> ret = new ArrayList<>();
342
343    for (E env: coprocEnvironments) {
344      C cp = env.getInstance();
345
346      if(cp != null) {
347        if (cls.isAssignableFrom(cp.getClass())) {
348          ret.add((T)cp);
349        }
350      }
351    }
352    return ret;
353  }
354
355  /**
356   * Find a coprocessor environment by class name
357   * @param className the class name
358   * @return the coprocessor, or null if not found
359   */
360  public E findCoprocessorEnvironment(String className) {
361    for (E env: coprocEnvironments) {
362      if (env.getInstance().getClass().getName().equals(className) ||
363          env.getInstance().getClass().getSimpleName().equals(className)) {
364        return env;
365      }
366    }
367    return null;
368  }
369
370  /**
371   * Retrieves the set of classloaders used to instantiate Coprocessor classes defined in external
372   * jar files.
373   * @return A set of ClassLoader instances
374   */
375  Set<ClassLoader> getExternalClassLoaders() {
376    Set<ClassLoader> externalClassLoaders = new HashSet<>();
377    final ClassLoader systemClassLoader = this.getClass().getClassLoader();
378    for (E env : coprocEnvironments) {
379      ClassLoader cl = env.getInstance().getClass().getClassLoader();
380      if (cl != systemClassLoader){
381        //do not include system classloader
382        externalClassLoaders.add(cl);
383      }
384    }
385    return externalClassLoaders;
386  }
387
388  /**
389   * Environment priority comparator.
390   * Coprocessors are chained in sorted order.
391   */
392  static class EnvironmentPriorityComparator implements Comparator<CoprocessorEnvironment> {
393    @Override
394    public int compare(final CoprocessorEnvironment env1,
395        final CoprocessorEnvironment env2) {
396      if (env1.getPriority() < env2.getPriority()) {
397        return -1;
398      } else if (env1.getPriority() > env2.getPriority()) {
399        return 1;
400      }
401      if (env1.getLoadSequence() < env2.getLoadSequence()) {
402        return -1;
403      } else if (env1.getLoadSequence() > env2.getLoadSequence()) {
404        return 1;
405      }
406      return 0;
407    }
408  }
409
410  protected void abortServer(final E environment, final Throwable e) {
411    abortServer(environment.getInstance().getClass().getName(), e);
412  }
413
414  protected void abortServer(final String coprocessorName, final Throwable e) {
415    String message = "The coprocessor " + coprocessorName + " threw " + e.toString();
416    LOG.error(message, e);
417    if (abortable != null) {
418      abortable.abort(message, e);
419    } else {
420      LOG.warn("No available Abortable, process was not aborted");
421    }
422  }
423
424  /**
425   * This is used by coprocessor hooks which are declared to throw IOException
426   * (or its subtypes). For such hooks, we should handle throwable objects
427   * depending on the Throwable's type. Those which are instances of
428   * IOException should be passed on to the client. This is in conformance with
429   * the HBase idiom regarding IOException: that it represents a circumstance
430   * that should be passed along to the client for its own handling. For
431   * example, a coprocessor that implements access controls would throw a
432   * subclass of IOException, such as AccessDeniedException, in its preGet()
433   * method to prevent an unauthorized client's performing a Get on a particular
434   * table.
435   * @param env Coprocessor Environment
436   * @param e Throwable object thrown by coprocessor.
437   * @exception IOException Exception
438   */
439  // Note to devs: Class comments of all observers ({@link MasterObserver}, {@link WALObserver},
440  // etc) mention this nuance of our exception handling so that coprocessor can throw appropriate
441  // exceptions depending on situation. If any changes are made to this logic, make sure to
442  // update all classes' comments.
443  protected void handleCoprocessorThrowable(final E env, final Throwable e) throws IOException {
444    if (e instanceof IOException) {
445      throw (IOException)e;
446    }
447    // If we got here, e is not an IOException. A loaded coprocessor has a
448    // fatal bug, and the server (master or regionserver) should remove the
449    // faulty coprocessor from its set of active coprocessors. Setting
450    // 'hbase.coprocessor.abortonerror' to true will cause abortServer(),
451    // which may be useful in development and testing environments where
452    // 'failing fast' for error analysis is desired.
453    if (env.getConfiguration().getBoolean(ABORT_ON_ERROR_KEY, DEFAULT_ABORT_ON_ERROR)) {
454      // server is configured to abort.
455      abortServer(env, e);
456    } else {
457      // If available, pull a table name out of the environment
458      if(env instanceof RegionCoprocessorEnvironment) {
459        String tableName = ((RegionCoprocessorEnvironment)env).getRegionInfo().getTable().getNameAsString();
460        LOG.error("Removing coprocessor '" + env.toString() + "' from table '"+ tableName + "'", e);
461      } else {
462        LOG.error("Removing coprocessor '" + env.toString() + "' from " +
463                "environment",e);
464      }
465
466      coprocEnvironments.remove(env);
467      try {
468        shutdown(env);
469      } catch (Exception x) {
470        LOG.error("Uncaught exception when shutting down coprocessor '"
471            + env.toString() + "'", x);
472      }
473      throw new DoNotRetryIOException("Coprocessor: '" + env.toString() +
474          "' threw: '" + e + "' and has been removed from the active " +
475          "coprocessor set.", e);
476    }
477  }
478
479  /**
480   * Used to limit legacy handling to once per Coprocessor class per classloader.
481   */
482  private static final Set<Class<? extends Coprocessor>> legacyWarning =
483      new ConcurrentSkipListSet<>(
484          new Comparator<Class<? extends Coprocessor>>() {
485            @Override
486            public int compare(Class<? extends Coprocessor> c1, Class<? extends Coprocessor> c2) {
487              if (c1.equals(c2)) {
488                return 0;
489              }
490              return c1.getName().compareTo(c2.getName());
491            }
492          });
493
494  /**
495   * Implementations defined function to get an observer of type {@code O} from a coprocessor of
496   * type {@code C}. Concrete implementations of CoprocessorHost define one getter for each
497   * observer they can handle. For e.g. RegionCoprocessorHost will use 3 getters, one for
498   * each of RegionObserver, EndpointObserver and BulkLoadObserver.
499   * These getters are used by {@code ObserverOperation} to get appropriate observer from the
500   * coprocessor.
501   */
502  @FunctionalInterface
503  public interface ObserverGetter<C, O> extends Function<C, Optional<O>> {}
504
505  private abstract class ObserverOperation<O> extends ObserverContextImpl<E> {
506    ObserverGetter<C, O> observerGetter;
507
508    ObserverOperation(ObserverGetter<C, O> observerGetter) {
509      this(observerGetter, null);
510    }
511
512    ObserverOperation(ObserverGetter<C, O> observerGetter, User user) {
513      this(observerGetter, user, false);
514    }
515
516    ObserverOperation(ObserverGetter<C, O> observerGetter, boolean bypassable) {
517      this(observerGetter, null, bypassable);
518    }
519
520    ObserverOperation(ObserverGetter<C, O> observerGetter, User user, boolean bypassable) {
521      super(user != null? user: RpcServer.getRequestUser().orElse(null), bypassable);
522      this.observerGetter = observerGetter;
523    }
524
525    abstract void callObserver() throws IOException;
526    protected void postEnvCall() {}
527  }
528
529  // Can't derive ObserverOperation from ObserverOperationWithResult (R = Void) because then all
530  // ObserverCaller implementations will have to have a return statement.
531  // O = observer, E = environment, C = coprocessor, R=result type
532  public abstract class ObserverOperationWithoutResult<O> extends ObserverOperation<O> {
533    protected abstract void call(O observer) throws IOException;
534
535    public ObserverOperationWithoutResult(ObserverGetter<C, O> observerGetter) {
536      super(observerGetter);
537    }
538
539    public ObserverOperationWithoutResult(ObserverGetter<C, O> observerGetter, User user) {
540      super(observerGetter, user);
541    }
542
543    public ObserverOperationWithoutResult(ObserverGetter<C, O> observerGetter, User user,
544        boolean bypassable) {
545      super(observerGetter, user, bypassable);
546    }
547
548    /**
549     * In case of coprocessors which have many kinds of observers (for eg, {@link RegionCoprocessor}
550     * has BulkLoadObserver, RegionObserver, etc), some implementations may not need all
551     * observers, in which case they will return null for that observer's getter.
552     * We simply ignore such cases.
553     */
554    @Override
555    void callObserver() throws IOException {
556      Optional<O> observer = observerGetter.apply(getEnvironment().getInstance());
557      if (observer.isPresent()) {
558        call(observer.get());
559      }
560    }
561  }
562
563  public abstract class ObserverOperationWithResult<O, R> extends ObserverOperation<O> {
564    protected abstract R call(O observer) throws IOException;
565
566    private R result;
567
568    public ObserverOperationWithResult(ObserverGetter<C, O> observerGetter, R result) {
569      this(observerGetter, result, false);
570    }
571
572    public ObserverOperationWithResult(ObserverGetter<C, O> observerGetter, R result,
573        boolean bypassable) {
574      this(observerGetter, result, null, bypassable);
575    }
576
577    public ObserverOperationWithResult(ObserverGetter<C, O> observerGetter, R result,
578        User user) {
579      this(observerGetter, result, user, false);
580    }
581
582    private ObserverOperationWithResult(ObserverGetter<C, O> observerGetter, R result, User user,
583        boolean bypassable) {
584      super(observerGetter, user, bypassable);
585      this.result = result;
586    }
587
588    protected R getResult() {
589      return this.result;
590    }
591
592    @Override
593    void callObserver() throws IOException {
594      Optional<O> observer = observerGetter.apply(getEnvironment().getInstance());
595      if (observer.isPresent()) {
596        result = call(observer.get());
597      }
598    }
599  }
600
601  //////////////////////////////////////////////////////////////////////////////////////////
602  // Functions to execute observer hooks and handle results (if any)
603  //////////////////////////////////////////////////////////////////////////////////////////
604
605  /**
606   * Do not call with an observerOperation that is null! Have the caller check.
607   */
608  protected <O, R> R execOperationWithResult(
609      final ObserverOperationWithResult<O, R> observerOperation) throws IOException {
610    boolean bypass = execOperation(observerOperation);
611    R result = observerOperation.getResult();
612    return bypass == observerOperation.isBypassable()? result: null;
613  }
614
615  /**
616   * @return True if we are to bypass (Can only be <code>true</code> if
617   * ObserverOperation#isBypassable().
618   */
619  protected <O> boolean execOperation(final ObserverOperation<O> observerOperation)
620      throws IOException {
621    boolean bypass = false;
622    if (observerOperation == null) {
623      return bypass;
624    }
625    List<E> envs = coprocEnvironments.get();
626    for (E env : envs) {
627      observerOperation.prepare(env);
628      Thread currentThread = Thread.currentThread();
629      ClassLoader cl = currentThread.getContextClassLoader();
630      try {
631        currentThread.setContextClassLoader(env.getClassLoader());
632        observerOperation.callObserver();
633      } catch (Throwable e) {
634        handleCoprocessorThrowable(env, e);
635      } finally {
636        currentThread.setContextClassLoader(cl);
637      }
638      // Internal to shouldBypass, it checks if obeserverOperation#isBypassable().
639      bypass |= observerOperation.shouldBypass();
640      observerOperation.postEnvCall();
641      if (bypass) {
642        // If CP says bypass, skip out w/o calling any following CPs; they might ruin our response.
643        // In hbase1, this used to be called 'complete'. In hbase2, we unite bypass and 'complete'.
644        break;
645      }
646    }
647    return bypass;
648  }
649
650  /**
651   * Coprocessor classes can be configured in any order, based on that priority is set and
652   * chained in a sorted order. Should be used preStop*() hooks i.e. when master/regionserver is
653   * going down. This function first calls coprocessor methods (using ObserverOperation.call())
654   * and then shutdowns the environment in postEnvCall(). <br>
655   * Need to execute all coprocessor methods first then postEnvCall(), otherwise some coprocessors
656   * may remain shutdown if any exception occurs during next coprocessor execution which prevent
657   * master/regionserver stop or cluster shutdown. (Refer:
658   * <a href="https://issues.apache.org/jira/browse/HBASE-16663">HBASE-16663</a>
659   * @return true if bypaas coprocessor execution, false if not.
660   * @throws IOException
661   */
662  protected <O> boolean execShutdown(final ObserverOperation<O> observerOperation)
663      throws IOException {
664    if (observerOperation == null) return false;
665    boolean bypass = false;
666    List<E> envs = coprocEnvironments.get();
667    // Iterate the coprocessors and execute ObserverOperation's call()
668    for (E env : envs) {
669      observerOperation.prepare(env);
670      Thread currentThread = Thread.currentThread();
671      ClassLoader cl = currentThread.getContextClassLoader();
672      try {
673        currentThread.setContextClassLoader(env.getClassLoader());
674        observerOperation.callObserver();
675      } catch (Throwable e) {
676        handleCoprocessorThrowable(env, e);
677      } finally {
678        currentThread.setContextClassLoader(cl);
679      }
680      bypass |= observerOperation.shouldBypass();
681    }
682
683    // Iterate the coprocessors and execute ObserverOperation's postEnvCall()
684    for (E env : envs) {
685      observerOperation.prepare(env);
686      observerOperation.postEnvCall();
687    }
688    return bypass;
689  }
690}