1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import java.io.File;
21 import java.io.FileOutputStream;
22 import java.io.IOException;
23 import java.net.URL;
24 import java.security.AccessController;
25 import java.security.PrivilegedAction;
26 import java.util.Collection;
27 import java.util.Enumeration;
28 import java.util.HashSet;
29 import java.util.concurrent.ConcurrentMap;
30 import java.util.concurrent.locks.Lock;
31 import java.util.jar.JarEntry;
32 import java.util.jar.JarFile;
33 import java.util.regex.Matcher;
34 import java.util.regex.Pattern;
35
36 import org.apache.commons.logging.Log;
37 import org.apache.commons.logging.LogFactory;
38 import org.apache.hadoop.hbase.classification.InterfaceAudience;
39 import org.apache.hadoop.conf.Configuration;
40 import org.apache.hadoop.fs.FileSystem;
41 import org.apache.hadoop.fs.Path;
42 import org.apache.hadoop.io.IOUtils;
43
44 import com.google.common.base.Preconditions;
45 import com.google.common.collect.MapMaker;
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74 @InterfaceAudience.Private
75 public class CoprocessorClassLoader extends ClassLoaderBase {
76 private static final Log LOG = LogFactory.getLog(CoprocessorClassLoader.class);
77
78
79
80 private static final String TMP_JARS_DIR = File.separator
81 + "jars" + File.separator + "tmp" + File.separator;
82
83
84
85
86
87
88
89 private static final ConcurrentMap<Path, CoprocessorClassLoader> classLoadersCache =
90 new MapMaker().concurrencyLevel(3).weakValues().makeMap();
91
92
93
94
95
96
97 private static final String[] CLASS_PREFIX_EXEMPTIONS = new String[] {
98
99 "com.sun.",
100 "launcher.",
101 "java.",
102 "javax.",
103 "org.ietf",
104 "org.omg",
105 "org.w3c",
106 "org.xml",
107 "sunw.",
108
109 "org.apache.commons.logging",
110 "org.apache.log4j",
111 "com.hadoop",
112
113 "org.apache.hadoop",
114 "org.apache.zookeeper",
115 };
116
117
118
119
120
121
122 private static final Pattern[] RESOURCE_LOAD_PARENT_FIRST_PATTERNS =
123 new Pattern[] {
124 Pattern.compile("^[^-]+-default\\.xml$")
125 };
126
127 private static final Pattern libJarPattern = Pattern.compile("[/]?lib/([^/]+\\.jar)");
128
129
130
131
132 private static final KeyLocker<String> locker = new KeyLocker<String>();
133
134
135
136
137
138 static final HashSet<String> parentDirLockSet = new HashSet<String>();
139
140
141
142
143 private CoprocessorClassLoader(ClassLoader parent) {
144 super(parent);
145 }
146
147 private void init(Path path, String pathPrefix,
148 Configuration conf) throws IOException {
149
150 String parentDirStr =
151 conf.get(LOCAL_DIR_KEY, DEFAULT_LOCAL_DIR) + TMP_JARS_DIR;
152 synchronized (parentDirLockSet) {
153 if (!parentDirLockSet.contains(parentDirStr)) {
154 Path parentDir = new Path(parentDirStr);
155 FileSystem fs = FileSystem.getLocal(conf);
156 fs.delete(parentDir, true);
157 parentDirLockSet.add(parentDirStr);
158 if (!fs.mkdirs(parentDir) && !fs.getFileStatus(parentDir).isDirectory()) {
159 throw new RuntimeException("Failed to create local dir " + parentDirStr
160 + ", CoprocessorClassLoader failed to init");
161 }
162 }
163 }
164
165 FileSystem fs = path.getFileSystem(conf);
166 File dst = new File(parentDirStr, "." + pathPrefix + "."
167 + path.getName() + "." + System.currentTimeMillis() + ".jar");
168 fs.copyToLocalFile(path, new Path(dst.toString()));
169 dst.deleteOnExit();
170
171 addURL(dst.getCanonicalFile().toURI().toURL());
172
173 JarFile jarFile = new JarFile(dst.toString());
174 try {
175 Enumeration<JarEntry> entries = jarFile.entries();
176 while (entries.hasMoreElements()) {
177 JarEntry entry = entries.nextElement();
178 Matcher m = libJarPattern.matcher(entry.getName());
179 if (m.matches()) {
180 File file = new File(parentDirStr, "." + pathPrefix + "."
181 + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
182 IOUtils.copyBytes(jarFile.getInputStream(entry),
183 new FileOutputStream(file), conf, true);
184 file.deleteOnExit();
185 addURL(file.toURI().toURL());
186 }
187 }
188 } finally {
189 jarFile.close();
190 }
191 }
192
193
194 public static CoprocessorClassLoader getIfCached(final Path path) {
195 Preconditions.checkNotNull(path, "The jar path is null!");
196 return classLoadersCache.get(path);
197 }
198
199
200 public static Collection<? extends ClassLoader> getAllCached() {
201 return classLoadersCache.values();
202 }
203
204
205 public static void clearCache() {
206 classLoadersCache.clear();
207 }
208
209
210
211
212
213
214
215
216
217
218
219
220 public static CoprocessorClassLoader getClassLoader(final Path path,
221 final ClassLoader parent, final String pathPrefix,
222 final Configuration conf) throws IOException {
223 CoprocessorClassLoader cl = getIfCached(path);
224 String pathStr = path.toString();
225 if (cl != null) {
226 LOG.debug("Found classloader "+ cl + " for "+ pathStr);
227 return cl;
228 }
229
230 if (!pathStr.endsWith(".jar")) {
231 throw new IOException(pathStr + ": not a jar file?");
232 }
233
234 Lock lock = locker.acquireLock(pathStr);
235 try {
236 cl = getIfCached(path);
237 if (cl != null) {
238 LOG.debug("Found classloader "+ cl + " for "+ pathStr);
239 return cl;
240 }
241
242 cl = AccessController.doPrivileged(
243 new PrivilegedAction<CoprocessorClassLoader>() {
244 @Override
245 public CoprocessorClassLoader run() {
246 return new CoprocessorClassLoader(parent);
247 }
248 });
249
250 cl.init(path, pathPrefix, conf);
251
252
253 CoprocessorClassLoader prev = classLoadersCache.putIfAbsent(path, cl);
254 if (prev != null) {
255
256 LOG.warn("THIS SHOULD NOT HAPPEN, a class loader"
257 +" is already cached for " + pathStr);
258 cl = prev;
259 }
260 return cl;
261 } finally {
262 lock.unlock();
263 }
264 }
265
266 @Override
267 public Class<?> loadClass(String name)
268 throws ClassNotFoundException {
269
270 if (isClassExempt(name)) {
271 if (LOG.isDebugEnabled()) {
272 LOG.debug("Skipping exempt class " + name +
273 " - delegating directly to parent");
274 }
275 return parent.loadClass(name);
276 }
277
278 synchronized (getClassLoadingLock(name)) {
279
280 Class<?> clasz = findLoadedClass(name);
281 if (clasz != null) {
282 if (LOG.isDebugEnabled()) {
283 LOG.debug("Class " + name + " already loaded");
284 }
285 }
286 else {
287 try {
288
289 if (LOG.isDebugEnabled()) {
290 LOG.debug("Finding class: " + name);
291 }
292 clasz = findClass(name);
293 } catch (ClassNotFoundException e) {
294
295 if (LOG.isDebugEnabled()) {
296 LOG.debug("Class " + name + " not found - delegating to parent");
297 }
298 try {
299 clasz = parent.loadClass(name);
300 } catch (ClassNotFoundException e2) {
301
302
303 if (LOG.isDebugEnabled()) {
304 LOG.debug("Class " + name + " not found in parent loader");
305 }
306 throw e2;
307 }
308 }
309 }
310 return clasz;
311 }
312 }
313
314 @Override
315 public URL getResource(String name) {
316 URL resource = null;
317 boolean parentLoaded = false;
318
319
320 if (loadResourceUsingParentFirst(name)) {
321 if (LOG.isDebugEnabled()) {
322 LOG.debug("Checking parent first for resource " + name);
323 }
324 resource = super.getResource(name);
325 parentLoaded = true;
326 }
327
328 if (resource == null) {
329 synchronized (getClassLoadingLock(name)) {
330
331 resource = findResource(name);
332 if ((resource == null) && !parentLoaded) {
333
334
335 resource = super.getResource(name);
336 }
337 }
338 }
339 return resource;
340 }
341
342
343
344
345
346
347
348
349 protected boolean isClassExempt(String name) {
350 for (String exemptPrefix : CLASS_PREFIX_EXEMPTIONS) {
351 if (name.startsWith(exemptPrefix)) {
352 return true;
353 }
354 }
355 return false;
356 }
357
358
359
360
361
362
363
364
365
366 protected boolean loadResourceUsingParentFirst(String name) {
367 for (Pattern resourcePattern : RESOURCE_LOAD_PARENT_FIRST_PATTERNS) {
368 if (resourcePattern.matcher(name).matches()) {
369 return true;
370 }
371 }
372 return false;
373 }
374 }