001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import java.io.File;
021import java.io.FileNotFoundException;
022import java.io.FileOutputStream;
023import java.io.IOException;
024import java.net.URL;
025import java.security.AccessController;
026import java.security.PrivilegedAction;
027import java.util.Collection;
028import java.util.Enumeration;
029import java.util.HashSet;
030import java.util.concurrent.ConcurrentMap;
031import java.util.concurrent.locks.Lock;
032import java.util.jar.JarEntry;
033import java.util.jar.JarFile;
034import java.util.regex.Matcher;
035import java.util.regex.Pattern;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.fs.FileStatus;
038import org.apache.hadoop.fs.FileSystem;
039import org.apache.hadoop.fs.FileUtil;
040import org.apache.hadoop.fs.Path;
041import org.apache.hadoop.io.IOUtils;
042import org.apache.yetus.audience.InterfaceAudience;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
047import org.apache.hbase.thirdparty.com.google.common.collect.MapMaker;
048
049/**
050 * ClassLoader used to load classes for Coprocessor instances.
051 * <p>
052 * This ClassLoader always tries to load classes from the specified coprocessor jar first actually
053 * using URLClassLoader logic before delegating to the parent ClassLoader, thus avoiding dependency
054 * conflicts between HBase's classpath and classes in the coprocessor jar.
055 * <p>
056 * Certain classes are exempt from being loaded by this ClassLoader because it would prevent them
057 * from being cast to the equivalent classes in the region server. For example, the Coprocessor
058 * interface needs to be loaded by the region server's ClassLoader to prevent a ClassCastException
059 * when casting the coprocessor implementation.
060 * <p>
061 * A HDFS path can be used to specify the coprocessor jar. In this case, the jar will be copied to
062 * local at first under some folder under ${hbase.local.dir}/jars/tmp/. The local copy will be
063 * removed automatically when the HBase server instance is stopped.
064 * <p>
065 * This ClassLoader also handles resource loading. In most cases this ClassLoader will attempt to
066 * load resources from the coprocessor jar first before delegating to the parent. However, like in
067 * class loading, some resources need to be handled differently. For all of the Hadoop default
068 * configurations (e.g. hbase-default.xml) we will check the parent ClassLoader first to prevent
069 * issues such as failing the HBase default configuration version check.
070 */
071@InterfaceAudience.Private
072public class CoprocessorClassLoader extends ClassLoaderBase {
073  private static final Logger LOG = LoggerFactory.getLogger(CoprocessorClassLoader.class);
074
075  // A temporary place ${hbase.local.dir}/jars/tmp/ to store the local
076  // copy of the jar file and the libraries contained in the jar.
077  private static final String TMP_JARS_DIR =
078    File.separator + "jars" + File.separator + "tmp" + File.separator;
079
080  /**
081   * External class loaders cache keyed by external jar path. ClassLoader instance is stored as a
082   * weak-reference to allow GC'ing when it is not used (@see HBASE-7205)
083   */
084  private static final ConcurrentMap<Path, CoprocessorClassLoader> classLoadersCache =
085    new MapMaker().concurrencyLevel(3).weakValues().makeMap();
086
087  /**
088   * If the class being loaded starts with any of these strings, we will skip trying to load it from
089   * the coprocessor jar and instead delegate directly to the parent ClassLoader.
090   */
091  private static final String[] CLASS_PREFIX_EXEMPTIONS = new String[] {
092    // Java standard library:
093    "com.sun.", "java.", "javax.", "org.ietf", "org.omg", "org.w3c", "org.xml", "sunw.",
094    // logging
095    "org.slf4j", "org.apache.log4j", "com.hadoop",
096    // HBase:
097    "org.apache.hadoop.hbase", };
098
099  /**
100   * If the resource being loaded matches any of these patterns, we will first attempt to load the
101   * resource with the parent ClassLoader. Only if the resource is not found by the parent do we
102   * attempt to load it from the coprocessor jar.
103   */
104  private static final Pattern[] RESOURCE_LOAD_PARENT_FIRST_PATTERNS =
105    new Pattern[] { Pattern.compile("^[^-]+-default\\.xml$") };
106
107  private static final Pattern libJarPattern = Pattern.compile("[/]?lib/([^/]+\\.jar)");
108
109  /**
110   * A locker used to synchronize class loader initialization per coprocessor jar file
111   */
112  private static final KeyLocker<String> locker = new KeyLocker<>();
113
114  /**
115   * A set used to synchronized parent path clean up. Generally, there should be only one parent
116   * path, but using a set so that we can support more.
117   */
118  static final HashSet<String> parentDirLockSet = new HashSet<>();
119
120  /**
121   * Creates a JarClassLoader that loads classes from the given paths.
122   */
123  private CoprocessorClassLoader(ClassLoader parent) {
124    super(parent);
125  }
126
127  private void init(Path pathPattern, String pathPrefix, Configuration conf) throws IOException {
128    // Copy the jar to the local filesystem
129    String parentDirStr = conf.get(LOCAL_DIR_KEY, DEFAULT_LOCAL_DIR) + TMP_JARS_DIR;
130    synchronized (parentDirLockSet) {
131      if (!parentDirLockSet.contains(parentDirStr)) {
132        Path parentDir = new Path(parentDirStr);
133        FileSystem fs = FileSystem.getLocal(conf);
134        fs.delete(parentDir, true); // it's ok if the dir doesn't exist now
135        parentDirLockSet.add(parentDirStr);
136        if (!fs.mkdirs(parentDir) && !fs.getFileStatus(parentDir).isDirectory()) {
137          throw new RuntimeException("Failed to create local dir " + parentDirStr
138            + ", CoprocessorClassLoader failed to init");
139        }
140      }
141    }
142
143    FileSystem fs = pathPattern.getFileSystem(conf);
144    // append "*.jar" if a directory is specified
145    Path pathPattern1 = fs.isDirectory(pathPattern) ? new Path(pathPattern, "*.jar") : pathPattern;
146    // return all files that match the pattern
147    FileStatus[] fileStatuses = fs.globStatus(pathPattern1);
148    if (fileStatuses == null || fileStatuses.length == 0) {
149      // if no one matches
150      throw new FileNotFoundException(pathPattern1.toString());
151    } else {
152      boolean validFileEncountered = false;
153      // for each file that match the pattern
154      for (Path path : FileUtil.stat2Paths(fileStatuses)) {
155        if (fs.isFile(path)) {
156          // only process files, skip for directories
157          File dst = new File(parentDirStr, "." + pathPrefix + "." + path.getName() + "."
158            + EnvironmentEdgeManager.currentTime() + ".jar");
159          fs.copyToLocalFile(path, new Path(dst.toString()));
160          dst.deleteOnExit();
161
162          addURL(dst.getCanonicalFile().toURI().toURL());
163
164          JarFile jarFile = new JarFile(dst.toString());
165          try {
166            // get entries inside a jar file
167            Enumeration<JarEntry> entries = jarFile.entries();
168            while (entries.hasMoreElements()) {
169              JarEntry entry = entries.nextElement();
170              Matcher m = libJarPattern.matcher(entry.getName());
171              if (m.matches()) {
172                File file = new File(parentDirStr, "." + pathPrefix + "." + path.getName() + "."
173                  + EnvironmentEdgeManager.currentTime() + "." + m.group(1));
174                try (FileOutputStream outStream = new FileOutputStream(file)) {
175                  IOUtils.copyBytes(jarFile.getInputStream(entry), outStream, conf, true);
176                }
177                file.deleteOnExit();
178                addURL(file.toURI().toURL());
179              }
180            }
181          } finally {
182            jarFile.close();
183          }
184          // Set to true when encountering a file
185          validFileEncountered = true;
186        }
187      }
188      if (validFileEncountered == false) {
189        // all items returned by globStatus() are directories
190        throw new FileNotFoundException("No file found matching " + pathPattern1.toString());
191      }
192    }
193  }
194
195  // This method is used in unit test
196  public static CoprocessorClassLoader getIfCached(final Path path) {
197    Preconditions.checkNotNull(path, "The jar path is null!");
198    return classLoadersCache.get(path);
199  }
200
201  // This method is used in unit test
202  public static Collection<? extends ClassLoader> getAllCached() {
203    return classLoadersCache.values();
204  }
205
206  // This method is used in unit test
207  public static void clearCache() {
208    classLoadersCache.clear();
209  }
210
211  /**
212   * Get a CoprocessorClassLoader for a coprocessor jar path from cache. If not in cache, create
213   * one.
214   * @param path       the path to the coprocessor jar file to load classes from
215   * @param parent     the parent class loader for exempted classes
216   * @param pathPrefix a prefix used in temp path name to store the jar file locally
217   * @param conf       the configuration used to create the class loader, if needed
218   * @return a CoprocessorClassLoader for the coprocessor jar path
219   */
220  public static CoprocessorClassLoader getClassLoader(final Path path, final ClassLoader parent,
221    final String pathPrefix, final Configuration conf) throws IOException {
222    CoprocessorClassLoader cl = getIfCached(path);
223    String pathStr = path.toString();
224    if (cl != null) {
225      LOG.debug("Found classloader " + cl + " for " + pathStr);
226      return cl;
227    }
228
229    if (path.getFileSystem(conf).isFile(path) && !pathStr.endsWith(".jar")) {
230      throw new IOException(pathStr + ": not a jar file?");
231    }
232
233    Lock lock = locker.acquireLock(pathStr);
234    try {
235      cl = getIfCached(path);
236      if (cl != null) {
237        LOG.debug("Found classloader " + cl + " for " + pathStr);
238        return cl;
239      }
240
241      cl = AccessController.doPrivileged(new PrivilegedAction<CoprocessorClassLoader>() {
242        @Override
243        public CoprocessorClassLoader run() {
244          return new CoprocessorClassLoader(parent);
245        }
246      });
247
248      cl.init(path, pathPrefix, conf);
249
250      // Cache class loader as a weak value, will be GC'ed when no reference left
251      CoprocessorClassLoader prev = classLoadersCache.putIfAbsent(path, cl);
252      if (prev != null) {
253        // Lost update race, use already added class loader
254        LOG.warn("THIS SHOULD NOT HAPPEN, a class loader" + " is already cached for " + pathStr);
255        cl = prev;
256      }
257      return cl;
258    } finally {
259      lock.unlock();
260    }
261  }
262
263  @Override
264  public Class<?> loadClass(String name) throws ClassNotFoundException {
265    return loadClass(name, null);
266  }
267
268  public Class<?> loadClass(String name, String[] includedClassPrefixes)
269    throws ClassNotFoundException {
270    // Delegate to the parent immediately if this class is exempt
271    if (isClassExempt(name, includedClassPrefixes)) {
272      if (LOG.isDebugEnabled()) {
273        LOG.debug("Skipping exempt class " + name + " - delegating directly to parent");
274      }
275      return parent.loadClass(name);
276    }
277
278    synchronized (getClassLoadingLock(name)) {
279      // Check whether the class has already been loaded:
280      Class<?> clasz = findLoadedClass(name);
281      if (clasz != null) {
282        if (LOG.isDebugEnabled()) {
283          LOG.debug("Class " + name + " already loaded");
284        }
285      } else {
286        try {
287          // Try to find this class using the URLs passed to this ClassLoader
288          if (LOG.isDebugEnabled()) {
289            LOG.debug("Finding class: " + name);
290          }
291          clasz = findClass(name);
292        } catch (ClassNotFoundException e) {
293          // Class not found using this ClassLoader, so delegate to parent
294          if (LOG.isDebugEnabled()) {
295            LOG.debug("Class " + name + " not found - delegating to parent");
296          }
297          try {
298            clasz = parent.loadClass(name);
299          } catch (ClassNotFoundException e2) {
300            // Class not found in this ClassLoader or in the parent ClassLoader
301            // Log some debug output before re-throwing ClassNotFoundException
302            if (LOG.isDebugEnabled()) {
303              LOG.debug("Class " + name + " not found in parent loader");
304            }
305            throw e2;
306          }
307        }
308      }
309      return clasz;
310    }
311  }
312
313  @Override
314  public URL getResource(String name) {
315    URL resource = null;
316    boolean parentLoaded = false;
317
318    // Delegate to the parent first if necessary
319    if (loadResourceUsingParentFirst(name)) {
320      if (LOG.isDebugEnabled()) {
321        LOG.debug("Checking parent first for resource " + name);
322      }
323      resource = super.getResource(name);
324      parentLoaded = true;
325    }
326
327    if (resource == null) {
328      synchronized (getClassLoadingLock(name)) {
329        // Try to find the resource in this jar
330        resource = findResource(name);
331        if ((resource == null) && !parentLoaded) {
332          // Not found in this jar and we haven't attempted to load
333          // the resource in the parent yet; fall back to the parent
334          resource = super.getResource(name);
335        }
336      }
337    }
338    return resource;
339  }
340
341  /**
342   * Determines whether the given class should be exempt from being loaded by this ClassLoader.
343   * @param name the name of the class to test.
344   * @return true if the class should *not* be loaded by this ClassLoader; false otherwise.
345   */
346  protected boolean isClassExempt(String name, String[] includedClassPrefixes) {
347    if (includedClassPrefixes != null) {
348      for (String clsName : includedClassPrefixes) {
349        if (name.startsWith(clsName)) {
350          return false;
351        }
352      }
353    }
354    for (String exemptPrefix : CLASS_PREFIX_EXEMPTIONS) {
355      if (name.startsWith(exemptPrefix)) {
356        return true;
357      }
358    }
359    return false;
360  }
361
362  /**
363   * Determines whether we should attempt to load the given resource using the parent first before
364   * attempting to load the resource using this ClassLoader.
365   * @param name the name of the resource to test.
366   * @return true if we should attempt to load the resource using the parent first; false if we
367   *         should attempt to load the resource using this ClassLoader first.
368   */
369  protected boolean loadResourceUsingParentFirst(String name) {
370    for (Pattern resourcePattern : RESOURCE_LOAD_PARENT_FIRST_PATTERNS) {
371      if (resourcePattern.matcher(name).matches()) {
372        return true;
373      }
374    }
375    return false;
376  }
377}