View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.File;
21  import java.io.IOException;
22  import java.net.MalformedURLException;
23  import java.net.URL;
24  import java.util.HashMap;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.classification.InterfaceAudience;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileStatus;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.Path;
33  
34  /**
35   * This is a class loader that can load classes dynamically from new
36   * jar files under a configured folder. The paths to the jar files are
37   * converted to URLs, and URLClassLoader logic is actually used to load
38   * classes. This class loader always uses its parent class loader
39   * to load a class at first. Only if its parent class loader
40   * can not load a class, we will try to load it using the logic here.
41   * <p>
42   * The configured folder can be a HDFS path. In this case, the jar files
43   * under that folder will be copied to local at first under ${hbase.local.dir}/jars/.
44   * The local copy will be updated if the remote copy is updated, according to its
45   * last modified timestamp.
46   * <p>
47   * We can't unload a class already loaded. So we will use the existing
48   * jar files we already know to load any class which can't be loaded
49   * using the parent class loader. If we still can't load the class from
50   * the existing jar files, we will check if any new jar file is added,
51   * if so, we will load the new jar file and try to load the class again.
52   * If still failed, a class not found exception will be thrown.
53   * <p>
54   * Be careful in uploading new jar files and make sure all classes
55   * are consistent, otherwise, we may not be able to load your
56   * classes properly.
57   */
58  @InterfaceAudience.Private
59  public class DynamicClassLoader extends ClassLoaderBase {
60    private static final Log LOG =
61        LogFactory.getLog(DynamicClassLoader.class);
62  
63    // Dynamic jars are put under ${hbase.local.dir}/jars/
64    private static final String DYNAMIC_JARS_DIR = File.separator
65      + "jars" + File.separator;
66  
67    private static final String DYNAMIC_JARS_DIR_KEY = "hbase.dynamic.jars.dir";
68  
69    private File localDir;
70  
71    // FileSystem of the remote path, set only if remoteDir != null
72    private FileSystem remoteDirFs;
73    private Path remoteDir;
74  
75    // Last modified time of local jars
76    private HashMap<String, Long> jarModifiedTime;
77  
78    /**
79     * Creates a DynamicClassLoader that can load classes dynamically
80     * from jar files under a specific folder.
81     *
82     * @param conf the configuration for the cluster.
83     * @param parent the parent ClassLoader to set.
84     */
85    public DynamicClassLoader(
86        final Configuration conf, final ClassLoader parent) {
87      super(parent);
88  
89      jarModifiedTime = new HashMap<String, Long>();
90      String localDirPath = conf.get(
91        LOCAL_DIR_KEY, DEFAULT_LOCAL_DIR) + DYNAMIC_JARS_DIR;
92      localDir = new File(localDirPath);
93      if (!localDir.mkdirs() && !localDir.isDirectory()) {
94        throw new RuntimeException("Failed to create local dir " + localDir.getPath()
95          + ", DynamicClassLoader failed to init");
96      }
97  
98      String remotePath = conf.get(DYNAMIC_JARS_DIR_KEY);
99      if (remotePath == null || remotePath.equals(localDirPath)) {
100       remoteDir = null;  // ignore if it is the same as the local path
101     } else {
102       remoteDir = new Path(remotePath);
103       try {
104         remoteDirFs = remoteDir.getFileSystem(conf);
105       } catch (IOException ioe) {
106         LOG.warn("Failed to identify the fs of dir "
107           + remoteDir + ", ignored", ioe);
108         remoteDir = null;
109       }
110     }
111   }
112 
113   @Override
114   public Class<?> loadClass(String name)
115       throws ClassNotFoundException {
116     try {
117       return parent.loadClass(name);
118     } catch (ClassNotFoundException e) {
119       if (LOG.isDebugEnabled()) {
120         LOG.debug("Class " + name + " not found - using dynamical class loader");
121       }
122 
123       synchronized (getClassLoadingLock(name)) {
124         // Check whether the class has already been loaded:
125         Class<?> clasz = findLoadedClass(name);
126         if (clasz != null) {
127           if (LOG.isDebugEnabled()) {
128             LOG.debug("Class " + name + " already loaded");
129           }
130         }
131         else {
132           try {
133             if (LOG.isDebugEnabled()) {
134               LOG.debug("Finding class: " + name);
135             }
136             clasz = findClass(name);
137           } catch (ClassNotFoundException cnfe) {
138             // Load new jar files if any
139             if (LOG.isDebugEnabled()) {
140               LOG.debug("Loading new jar files, if any");
141             }
142             loadNewJars();
143 
144             if (LOG.isDebugEnabled()) {
145               LOG.debug("Finding class again: " + name);
146             }
147             clasz = findClass(name);
148           }
149         }
150         return clasz;
151       }
152     }
153   }
154 
155   private synchronized void loadNewJars() {
156     // Refresh local jar file lists
157     for (File file: localDir.listFiles()) {
158       String fileName = file.getName();
159       if (jarModifiedTime.containsKey(fileName)) {
160         continue;
161       }
162       if (file.isFile() && fileName.endsWith(".jar")) {
163         jarModifiedTime.put(fileName, Long.valueOf(file.lastModified()));
164         try {
165           URL url = file.toURI().toURL();
166           addURL(url);
167         } catch (MalformedURLException mue) {
168           // This should not happen, just log it
169           LOG.warn("Failed to load new jar " + fileName, mue);
170         }
171       }
172     }
173 
174     // Check remote files
175     FileStatus[] statuses = null;
176     if (remoteDir != null) {
177       try {
178         statuses = remoteDirFs.listStatus(remoteDir);
179       } catch (IOException ioe) {
180         LOG.warn("Failed to check remote dir status " + remoteDir, ioe);
181       }
182     }
183     if (statuses == null || statuses.length == 0) {
184       return; // no remote files at all
185     }
186 
187     for (FileStatus status: statuses) {
188       if (status.isDir()) continue; // No recursive lookup
189       Path path = status.getPath();
190       String fileName = path.getName();
191       if (!fileName.endsWith(".jar")) {
192         if (LOG.isDebugEnabled()) {
193           LOG.debug("Ignored non-jar file " + fileName);
194         }
195         continue; // Ignore non-jar files
196       }
197       Long cachedLastModificationTime = jarModifiedTime.get(fileName);
198       if (cachedLastModificationTime != null) {
199         long lastModified = status.getModificationTime();
200         if (lastModified < cachedLastModificationTime.longValue()) {
201           // There could be some race, for example, someone uploads
202           // a new one right in the middle the old one is copied to
203           // local. We can check the size as well. But it is still
204           // not guaranteed. This should be rare. Most likely,
205           // we already have the latest one.
206           // If you are unlucky to hit this race issue, you have
207           // to touch the remote jar to update its last modified time
208           continue;
209         }
210       }
211       try {
212         // Copy it to local
213         File dst = new File(localDir, fileName);
214         remoteDirFs.copyToLocalFile(path, new Path(dst.getPath()));
215         jarModifiedTime.put(fileName, Long.valueOf(dst.lastModified()));
216         URL url = dst.toURI().toURL();
217         addURL(url);
218       } catch (IOException ioe) {
219         LOG.warn("Failed to load new jar " + fileName, ioe);
220       }
221     }
222   }
223 }