View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.coprocessor;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.Collections;
25  import java.util.Comparator;
26  import java.util.HashSet;
27  import java.util.List;
28  import java.util.Set;
29  import java.util.TreeSet;
30  import java.util.UUID;
31  import java.util.concurrent.ConcurrentSkipListSet;
32  import java.util.concurrent.ExecutorService;
33  import java.util.concurrent.atomic.AtomicInteger;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.hbase.classification.InterfaceAudience;
38  import org.apache.hadoop.hbase.classification.InterfaceStability;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.fs.Path;
41  import org.apache.hadoop.hbase.Abortable;
42  import org.apache.hadoop.hbase.Coprocessor;
43  import org.apache.hadoop.hbase.CoprocessorEnvironment;
44  import org.apache.hadoop.hbase.DoNotRetryIOException;
45  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
46  import org.apache.hadoop.hbase.HConstants;
47  import org.apache.hadoop.hbase.TableName;
48  import org.apache.hadoop.hbase.client.HTable;
49  import org.apache.hadoop.hbase.client.HTableWrapper;
50  import org.apache.hadoop.hbase.client.Table;
51  import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
52  import org.apache.hadoop.hbase.util.SortedList;
53  import org.apache.hadoop.hbase.util.VersionInfo;
54
55  /**
56   * Provides the common setup framework and runtime services for coprocessor
57   * invocation from HBase services.
58   * @param <E> the specific environment extension that a concrete implementation
59   * provides
60   */
61  @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
62  @InterfaceStability.Evolving
63  public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
64    public static final String REGION_COPROCESSOR_CONF_KEY =
65        "hbase.coprocessor.region.classes";
66    public static final String REGIONSERVER_COPROCESSOR_CONF_KEY =
67        "hbase.coprocessor.regionserver.classes";
68    public static final String USER_REGION_COPROCESSOR_CONF_KEY =
69        "hbase.coprocessor.user.region.classes";
70    public static final String MASTER_COPROCESSOR_CONF_KEY =
71        "hbase.coprocessor.master.classes";
72    public static final String WAL_COPROCESSOR_CONF_KEY =
73      "hbase.coprocessor.wal.classes";
74    public static final String ABORT_ON_ERROR_KEY = "hbase.coprocessor.abortonerror";
75    public static final boolean DEFAULT_ABORT_ON_ERROR = true;
76    public static final String COPROCESSORS_ENABLED_CONF_KEY = "hbase.coprocessor.enabled";
77    public static final boolean DEFAULT_COPROCESSORS_ENABLED = true;
78    public static final String USER_COPROCESSORS_ENABLED_CONF_KEY =
79      "hbase.coprocessor.user.enabled";
80    public static final boolean DEFAULT_USER_COPROCESSORS_ENABLED = true;
81
82    private static final Log LOG = LogFactory.getLog(CoprocessorHost.class);
83    protected Abortable abortable;
84    /** Ordered set of loaded coprocessors with lock */
85    protected SortedList<E> coprocessors =
86        new SortedList<E>(new EnvironmentPriorityComparator());
87    protected Configuration conf;
88    // unique file prefix to use for local copies of jars when classloading
89    protected String pathPrefix;
90    protected AtomicInteger loadSequence = new AtomicInteger();
91
92    public CoprocessorHost(Abortable abortable) {
93      this.abortable = abortable;
94      this.pathPrefix = UUID.randomUUID().toString();
95    }
96
97    /**
98     * Not to be confused with the per-object _coprocessors_ (above),
99     * coprocessorNames is static and stores the set of all coprocessors ever
100    * loaded by any thread in this JVM. It is strictly additive: coprocessors are
101    * added to coprocessorNames, by loadInstance() but are never removed, since
102    * the intention is to preserve a history of all loaded coprocessors for
103    * diagnosis in case of server crash (HBASE-4014).
104    */
105   private static Set<String> coprocessorNames =
106       Collections.synchronizedSet(new HashSet<String>());
107
108   public static Set<String> getLoadedCoprocessors() {
109     synchronized (coprocessorNames) {
110       return new HashSet(coprocessorNames);
111     }
112   }
113
114   /**
115    * Used to create a parameter to the HServerLoad constructor so that
116    * HServerLoad can provide information about the coprocessors loaded by this
117    * regionserver.
118    * (HBASE-4070: Improve region server metrics to report loaded coprocessors
119    * to master).
120    */
121   public Set<String> getCoprocessors() {
122     Set<String> returnValue = new TreeSet<String>();
123     for (CoprocessorEnvironment e: coprocessors) {
124       returnValue.add(e.getInstance().getClass().getSimpleName());
125     }
126     return returnValue;
127   }
128
129   /**
130    * Load system coprocessors once only. Read the class names from configuration.
131    * Called by constructor.
132    */
133   protected void loadSystemCoprocessors(Configuration conf, String confKey) {
134     boolean coprocessorsEnabled = conf.getBoolean(COPROCESSORS_ENABLED_CONF_KEY,
135       DEFAULT_COPROCESSORS_ENABLED);
136     if (!coprocessorsEnabled) {
137       return;
138     }
139
140     Class<?> implClass = null;
141
142     // load default coprocessors from configure file
143     String[] defaultCPClasses = conf.getStrings(confKey);
144     if (defaultCPClasses == null || defaultCPClasses.length == 0)
145       return;
146
147     int priority = Coprocessor.PRIORITY_SYSTEM;
148     for (String className : defaultCPClasses) {
149       className = className.trim();
150       if (findCoprocessor(className) != null) {
151         // If already loaded will just continue
152         LOG.warn("Attempted duplicate loading of " + className + "; skipped");
153         continue;
154       }
155       ClassLoader cl = this.getClass().getClassLoader();
156       Thread.currentThread().setContextClassLoader(cl);
157       try {
158         implClass = cl.loadClass(className);
159         // Add coprocessors as we go to guard against case where a coprocessor is specified twice
160         // in the configuration
161         this.coprocessors.add(loadInstance(implClass, Coprocessor.PRIORITY_SYSTEM, conf));
162         LOG.info("System coprocessor " + className + " was loaded " +
163             "successfully with priority (" + priority++ + ").");
164       } catch (Throwable t) {
165         // We always abort if system coprocessors cannot be loaded
166         abortServer(className, t);
167       }
168     }
169   }
170
171   /**
172    * Load a coprocessor implementation into the host
173    * @param path path to implementation jar
174    * @param className the main class name
175    * @param priority chaining priority
176    * @param conf configuration for coprocessor
177    * @throws java.io.IOException Exception
178    */
179   public E load(Path path, String className, int priority,
180       Configuration conf) throws IOException {
181     String[] includedClassPrefixes = null;
182     if (conf.get(HConstants.CP_HTD_ATTR_INCLUSION_KEY) != null){
183       String prefixes = conf.get(HConstants.CP_HTD_ATTR_INCLUSION_KEY);
184       includedClassPrefixes = prefixes.split(";");
185     }
186     return load(path, className, priority, conf, includedClassPrefixes);
187   }
188
189   /**
190    * Load a coprocessor implementation into the host
191    * @param path path to implementation jar
192    * @param className the main class name
193    * @param priority chaining priority
194    * @param conf configuration for coprocessor
195    * @param includedClassPrefixes class name prefixes to include
196    * @throws java.io.IOException Exception
197    */
198   public E load(Path path, String className, int priority,
199       Configuration conf, String[] includedClassPrefixes) throws IOException {
200     Class<?> implClass = null;
201     LOG.debug("Loading coprocessor class " + className + " with path " +
202         path + " and priority " + priority);
203
204     ClassLoader cl = null;
205     if (path == null) {
206       try {
207         implClass = getClass().getClassLoader().loadClass(className);
208       } catch (ClassNotFoundException e) {
209         throw new IOException("No jar path specified for " + className);
210       }
211     } else {
212       cl = CoprocessorClassLoader.getClassLoader(
213         path, getClass().getClassLoader(), pathPrefix, conf);
214       try {
215         implClass = ((CoprocessorClassLoader)cl).loadClass(className, includedClassPrefixes);
216       } catch (ClassNotFoundException e) {
217         throw new IOException("Cannot load external coprocessor class " + className, e);
218       }
219     }
220
221     //load custom code for coprocessor
222     Thread currentThread = Thread.currentThread();
223     ClassLoader hostClassLoader = currentThread.getContextClassLoader();
224     try{
225       // switch temporarily to the thread classloader for custom CP
226       currentThread.setContextClassLoader(cl);
227       E cpInstance = loadInstance(implClass, priority, conf);
228       return cpInstance;
229     } finally {
230       // restore the fresh (host) classloader
231       currentThread.setContextClassLoader(hostClassLoader);
232     }
233   }
234
235   /**
236    * @param implClass Implementation class
237    * @param priority priority
238    * @param conf configuration
239    * @throws java.io.IOException Exception
240    */
241   public void load(Class<?> implClass, int priority, Configuration conf)
242       throws IOException {
243     E env = loadInstance(implClass, priority, conf);
244     coprocessors.add(env);
245   }
246
247   /**
248    * @param implClass Implementation class
249    * @param priority priority
250    * @param conf configuration
251    * @throws java.io.IOException Exception
252    */
253   public E loadInstance(Class<?> implClass, int priority, Configuration conf)
254       throws IOException {
255     if (!Coprocessor.class.isAssignableFrom(implClass)) {
256       throw new IOException("Configured class " + implClass.getName() + " must implement "
257           + Coprocessor.class.getName() + " interface ");
258     }
259
260     // create the instance
261     Coprocessor impl;
262     Object o = null;
263     try {
264       o = implClass.newInstance();
265       impl = (Coprocessor)o;
266     } catch (InstantiationException e) {
267       throw new IOException(e);
268     } catch (IllegalAccessException e) {
269       throw new IOException(e);
270     }
271     // create the environment
272     E env = createEnvironment(implClass, impl, priority, loadSequence.incrementAndGet(), conf);
273     if (env instanceof Environment) {
274       ((Environment)env).startup();
275     }
276     // HBASE-4014: maintain list of loaded coprocessors for later crash analysis
277     // if server (master or regionserver) aborts.
278     coprocessorNames.add(implClass.getName());
279     return env;
280   }
281
282   /**
283    * Called when a new Coprocessor class is loaded
284    */
285   public abstract E createEnvironment(Class<?> implClass, Coprocessor instance,
286       int priority, int sequence, Configuration conf);
287
288   public void shutdown(CoprocessorEnvironment e) {
289     if (e instanceof Environment) {
290       if (LOG.isDebugEnabled()) {
291         LOG.debug("Stop coprocessor " + e.getInstance().getClass().getName());
292       }
293       ((Environment)e).shutdown();
294     } else {
295       LOG.warn("Shutdown called on unknown environment: "+
296           e.getClass().getName());
297     }
298   }
299
300   /**
301    * Find a coprocessor implementation by class name
302    * @param className the class name
303    * @return the coprocessor, or null if not found
304    */
305   public Coprocessor findCoprocessor(String className) {
306     for (E env: coprocessors) {
307       if (env.getInstance().getClass().getName().equals(className) ||
308           env.getInstance().getClass().getSimpleName().equals(className)) {
309         return env.getInstance();
310       }
311     }
312     return null;
313   }
314 
315   /**
316    * Find list of coprocessors that extend/implement the given class/interface
317    * @param cls the class/interface to look for
318    * @return the list of coprocessors, or null if not found
319    */
320   public <T extends Coprocessor> List<T> findCoprocessors(Class<T> cls) {
321     ArrayList<T> ret = new ArrayList<T>();
322
323     for (E env: coprocessors) {
324       Coprocessor cp = env.getInstance();
325
326       if(cp != null) {
327         if (cls.isAssignableFrom(cp.getClass())) {
328           ret.add((T)cp);
329         }
330       }
331     }
332     return ret;
333   }
334
335   /**
336    * Find a coprocessor environment by class name
337    * @param className the class name
338    * @return the coprocessor, or null if not found
339    */
340   public CoprocessorEnvironment findCoprocessorEnvironment(String className) {
341     for (E env: coprocessors) {
342       if (env.getInstance().getClass().getName().equals(className) ||
343           env.getInstance().getClass().getSimpleName().equals(className)) {
344         return env;
345       }
346     }
347     return null;
348   }
349
350   /**
351    * Retrieves the set of classloaders used to instantiate Coprocessor classes defined in external
352    * jar files.
353    * @return A set of ClassLoader instances
354    */
355   Set<ClassLoader> getExternalClassLoaders() {
356     Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>();
357     final ClassLoader systemClassLoader = this.getClass().getClassLoader();
358     for (E env : coprocessors) {
359       ClassLoader cl = env.getInstance().getClass().getClassLoader();
360       if (cl != systemClassLoader){
361         //do not include system classloader
362         externalClassLoaders.add(cl);
363       }
364     }
365     return externalClassLoaders;
366   }
367
368   /**
369    * Environment priority comparator.
370    * Coprocessors are chained in sorted order.
371    */
372   static class EnvironmentPriorityComparator
373       implements Comparator<CoprocessorEnvironment> {
374     @Override
375     public int compare(final CoprocessorEnvironment env1,
376         final CoprocessorEnvironment env2) {
377       if (env1.getPriority() < env2.getPriority()) {
378         return -1;
379       } else if (env1.getPriority() > env2.getPriority()) {
380         return 1;
381       }
382       if (env1.getLoadSequence() < env2.getLoadSequence()) {
383         return -1;
384       } else if (env1.getLoadSequence() > env2.getLoadSequence()) {
385         return 1;
386       }
387       return 0;
388     }
389   }
390
391   /**
392    * Encapsulation of the environment of each coprocessor
393    */
394   public static class Environment implements CoprocessorEnvironment {
395
396     /** The coprocessor */
397     public Coprocessor impl;
398     /** Chaining priority */
399     protected int priority = Coprocessor.PRIORITY_USER;
400     /** Current coprocessor state */
401     Coprocessor.State state = Coprocessor.State.UNINSTALLED;
402     /** Accounting for tables opened by the coprocessor */
403     protected List<Table> openTables =
404       Collections.synchronizedList(new ArrayList<Table>());
405     private int seq;
406     private Configuration conf;
407     private ClassLoader classLoader;
408
409     /**
410      * Constructor
411      * @param impl the coprocessor instance
412      * @param priority chaining priority
413      */
414     public Environment(final Coprocessor impl, final int priority,
415         final int seq, final Configuration conf) {
416       this.impl = impl;
417       this.classLoader = impl.getClass().getClassLoader();
418       this.priority = priority;
419       this.state = Coprocessor.State.INSTALLED;
420       this.seq = seq;
421       this.conf = conf;
422     }
423 
424     /** Initialize the environment */
425     public void startup() throws IOException {
426       if (state == Coprocessor.State.INSTALLED ||
427           state == Coprocessor.State.STOPPED) {
428         state = Coprocessor.State.STARTING;
429         Thread currentThread = Thread.currentThread();
430         ClassLoader hostClassLoader = currentThread.getContextClassLoader();
431         try {
432           currentThread.setContextClassLoader(this.getClassLoader());
433           impl.start(this);
434           state = Coprocessor.State.ACTIVE;
435         } finally {
436           currentThread.setContextClassLoader(hostClassLoader);
437         }
438       } else {
439         LOG.warn("Not starting coprocessor "+impl.getClass().getName()+
440             " because not inactive (state="+state.toString()+")");
441       }
442     }
443
444     /** Clean up the environment */
445     protected void shutdown() {
446       if (state == Coprocessor.State.ACTIVE) {
447         state = Coprocessor.State.STOPPING;
448         Thread currentThread = Thread.currentThread();
449         ClassLoader hostClassLoader = currentThread.getContextClassLoader();
450         try {
451           currentThread.setContextClassLoader(this.getClassLoader());
452           impl.stop(this);
453           state = Coprocessor.State.STOPPED;
454         } catch (IOException ioe) {
455           LOG.error("Error stopping coprocessor "+impl.getClass().getName(), ioe);
456         } finally {
457           currentThread.setContextClassLoader(hostClassLoader);
458         }
459       } else {
460         LOG.warn("Not stopping coprocessor "+impl.getClass().getName()+
461             " because not active (state="+state.toString()+")");
462       }
463       synchronized (openTables) {
464         // clean up any table references
465         for (Table table: openTables) {
466           try {
467             ((HTableWrapper)table).internalClose();
468           } catch (IOException e) {
469             // nothing can be done here
470             LOG.warn("Failed to close " +
471                 table.getName(), e);
472           }
473         }
474       }
475     }
476 
477     @Override
478     public Coprocessor getInstance() {
479       return impl;
480     }
481
482     @Override
483     public ClassLoader getClassLoader() {
484       return classLoader;
485     }
486
487     @Override
488     public int getPriority() {
489       return priority;
490     }
491
492     @Override
493     public int getLoadSequence() {
494       return seq;
495     }
496
497     /** @return the coprocessor environment version */
498     @Override
499     public int getVersion() {
500       return Coprocessor.VERSION;
501     }
502
503     /** @return the HBase release */
504     @Override
505     public String getHBaseVersion() {
506       return VersionInfo.getVersion();
507     }
508
509     @Override
510     public Configuration getConfiguration() {
511       return conf;
512     }
513
514     /**
515      * Open a table from within the Coprocessor environment
516      * @param tableName the table name
517      * @return an interface for manipulating the table
518      * @exception java.io.IOException Exception
519      */
520     @Override
521     public Table getTable(TableName tableName) throws IOException {
522       return this.getTable(tableName, HTable.getDefaultExecutor(getConfiguration()));
523     }
524
525     /**
526      * Open a table from within the Coprocessor environment
527      * @param tableName the table name
528      * @return an interface for manipulating the table
529      * @exception java.io.IOException Exception
530      */
531     @Override
532     public Table getTable(TableName tableName, ExecutorService pool) throws IOException {
533       return HTableWrapper.createWrapper(openTables, tableName, this, pool);
534     }
535   }
536
537   protected void abortServer(final CoprocessorEnvironment environment, final Throwable e) {
538     abortServer(environment.getInstance().getClass().getName(), e);
539   }
540
541   protected void abortServer(final String coprocessorName, final Throwable e) {
542     String message = "The coprocessor " + coprocessorName + " threw " + e.toString();
543     LOG.error(message, e);
544     if (abortable != null) {
545       abortable.abort(message, e);
546     } else {
547       LOG.warn("No available Abortable, process was not aborted");
548     }
549   }
550
551   /**
552    * This is used by coprocessor hooks which are declared to throw IOException
553    * (or its subtypes). For such hooks, we should handle throwable objects
554    * depending on the Throwable's type. Those which are instances of
555    * IOException should be passed on to the client. This is in conformance with
556    * the HBase idiom regarding IOException: that it represents a circumstance
557    * that should be passed along to the client for its own handling. For
558    * example, a coprocessor that implements access controls would throw a
559    * subclass of IOException, such as AccessDeniedException, in its preGet()
560    * method to prevent an unauthorized client's performing a Get on a particular
561    * table.
562    * @param env Coprocessor Environment
563    * @param e Throwable object thrown by coprocessor.
564    * @exception IOException Exception
565    */
566   protected void handleCoprocessorThrowable(final CoprocessorEnvironment env, final Throwable e)
567       throws IOException {
568     if (e instanceof IOException) {
569       throw (IOException)e;
570     }
571     // If we got here, e is not an IOException. A loaded coprocessor has a
572     // fatal bug, and the server (master or regionserver) should remove the
573     // faulty coprocessor from its set of active coprocessors. Setting
574     // 'hbase.coprocessor.abortonerror' to true will cause abortServer(),
575     // which may be useful in development and testing environments where
576     // 'failing fast' for error analysis is desired.
577     if (env.getConfiguration().getBoolean(ABORT_ON_ERROR_KEY, DEFAULT_ABORT_ON_ERROR)) {
578       // server is configured to abort.
579       abortServer(env, e);
580     } else {
581       LOG.error("Removing coprocessor '" + env.toString() + "' from " +
582           "environment because it threw:  " + e,e);
583       coprocessors.remove(env);
584       try {
585         shutdown(env);
586       } catch (Exception x) {
587         LOG.error("Uncaught exception when shutting down coprocessor '"
588             + env.toString() + "'", x);
589       }
590       throw new DoNotRetryIOException("Coprocessor: '" + env.toString() +
591           "' threw: '" + e + "' and has been removed from the active " +
592           "coprocessor set.", e);
593     }
594   }
595
596   /**
597    * Used to gracefully handle fallback to deprecated methods when we
598    * evolve coprocessor APIs.
599    *
600    * When a particular Coprocessor API is updated to change methods, hosts can support fallback
601    * to the deprecated API by using this method to determine if an instance implements the new API.
602    * In the event that said support is partial, then in the face of a runtime issue that prevents
603    * proper operation {@link #legacyWarning(Class, String)} should be used to let operators know.
604    *
605    * For examples of this in action, see the implementation of
606    * <ul>
607    *   <li>{@link org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost}
608    *   <li>{@link org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost}
609    * </ul>
610    *
611    * @param clazz Coprocessor you wish to evaluate
612    * @param methodName the name of the non-deprecated method version
613    * @param parameterTypes the Class of the non-deprecated method's arguments in the order they are
614    *     declared.
615    */
616   @InterfaceAudience.Private
617   protected static boolean useLegacyMethod(final Class<? extends Coprocessor> clazz,
618       final String methodName, final Class<?>... parameterTypes) {
619     boolean useLegacy;
620     // Use reflection to see if they implement the non-deprecated version
621     try {
622       clazz.getDeclaredMethod(methodName, parameterTypes);
623       LOG.debug("Found an implementation of '" + methodName + "' that uses updated method " +
624           "signature. Skipping legacy support for invocations in '" + clazz +"'.");
625       useLegacy = false;
626     } catch (NoSuchMethodException exception) {
627       useLegacy = true;
628     } catch (SecurityException exception) {
629       LOG.warn("The Security Manager denied our attempt to detect if the coprocessor '" + clazz +
630           "' requires legacy support; assuming it does. If you get later errors about legacy " +
631           "coprocessor use, consider updating your security policy to allow access to the package" +
632           " and declared members of your implementation.");
633       LOG.debug("Details of Security Manager rejection.", exception);
634       useLegacy = true;
635     }
636     return useLegacy;
637   }
638
639   /**
640    * Used to limit legacy handling to once per Coprocessor class per classloader.
641    */
642   private static final Set<Class<? extends Coprocessor>> legacyWarning =
643       new ConcurrentSkipListSet<Class<? extends Coprocessor>>(
644           new Comparator<Class<? extends Coprocessor>>() {
645             @Override
646             public int compare(Class<? extends Coprocessor> c1, Class<? extends Coprocessor> c2) {
647               if (c1.equals(c2)) {
648                 return 0;
649               }
650               return c1.getName().compareTo(c2.getName());
651             }
652           });
653
654   /**
655    * limits the amount of logging to once per coprocessor class.
656    * Used in concert with {@link #useLegacyMethod(Class, String, Class[])} when a runtime issue
657    * prevents properly supporting the legacy version of a coprocessor API.
658    * Since coprocessors can be in tight loops this serves to limit the amount of log spam we create.
659    */
660   @InterfaceAudience.Private
661   protected void legacyWarning(final Class<? extends Coprocessor> clazz, final String message) {
662     if(legacyWarning.add(clazz)) {
663       LOG.error("You have a legacy coprocessor loaded and there are events we can't map to the " +
664           " deprecated API. Your coprocessor will not see these events.  Please update '" + clazz +
665           "'. Details of the problem: " + message);
666     }
667   }
668 }