001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.Assert.assertArrayEquals;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024
025import java.io.File;
026import java.io.FileInputStream;
027import java.io.FileOutputStream;
028import java.io.IOException;
029import java.io.PrintStream;
030import java.net.URL;
031import java.net.URLClassLoader;
032import java.util.HashSet;
033import java.util.Set;
034import java.util.concurrent.atomic.AtomicLong;
035import java.util.jar.Attributes;
036import java.util.jar.JarEntry;
037import java.util.jar.JarOutputStream;
038import java.util.jar.Manifest;
039import javax.tools.JavaCompiler;
040import javax.tools.ToolProvider;
041import org.apache.hadoop.hbase.testclassification.MiscTests;
042import org.apache.hadoop.hbase.testclassification.SmallTests;
043import org.junit.AfterClass;
044import org.junit.BeforeClass;
045import org.junit.ClassRule;
046import org.junit.Rule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049import org.junit.rules.TestName;
050import org.slf4j.Logger;
051import org.slf4j.LoggerFactory;
052
053@Category({MiscTests.class, SmallTests.class})
054public class TestClassFinder {
055  @ClassRule
056  public static final HBaseClassTestRule CLASS_RULE =
057      HBaseClassTestRule.forClass(TestClassFinder.class);
058
059  private static final Logger LOG = LoggerFactory.getLogger(TestClassFinder.class);
060
061  @Rule public TestName name = new TestName();
062  private static final HBaseCommonTestingUtility testUtil = new HBaseCommonTestingUtility();
063  private static final String BASEPKG = "tfcpkg";
064  private static final String PREFIX = "Prefix";
065
066  // Use unique jar/class/package names in each test case with the help
067  // of these global counters; we are mucking with ClassLoader in this test
068  // and we don't want individual test cases to conflict via it.
069  private static AtomicLong testCounter = new AtomicLong(0);
070  private static AtomicLong jarCounter = new AtomicLong(0);
071
072  private static String basePath = null;
073
074  private static CustomClassloader classLoader;
075
076  @BeforeClass
077  public static void createTestDir() throws IOException {
078    basePath = testUtil.getDataTestDir(TestClassFinder.class.getSimpleName()).toString();
079    if (!basePath.endsWith("/")) {
080      basePath += "/";
081    }
082    // Make sure we get a brand new directory.
083    File testDir = new File(basePath);
084    if (testDir.exists()) {
085      deleteTestDir();
086    }
087    assertTrue(testDir.mkdirs());
088    LOG.info("Using new, clean directory=" + testDir);
089
090    classLoader = new CustomClassloader(new URL[0], ClassLoader.getSystemClassLoader());
091  }
092
093  @AfterClass
094  public static void deleteTestDir() {
095    testUtil.cleanupTestDir(TestClassFinder.class.getSimpleName());
096  }
097
098  @Test
099  public void testClassFinderCanFindClassesInJars() throws Exception {
100    long counter = testCounter.incrementAndGet();
101    FileAndPath c1 = compileTestClass(counter, "", "c1");
102    FileAndPath c2 = compileTestClass(counter, ".nested", "c2");
103    FileAndPath c3 = compileTestClass(counter, "", "c3");
104    packageAndLoadJar(c1, c3);
105    packageAndLoadJar(c2);
106
107    ClassFinder allClassesFinder = new ClassFinder(classLoader);
108    Set<Class<?>> allClasses = allClassesFinder.findClasses(
109        makePackageName("", counter), false);
110    assertEquals(3, allClasses.size());
111  }
112
113  @Test
114  public void testClassFinderHandlesConflicts() throws Exception {
115    long counter = testCounter.incrementAndGet();
116    FileAndPath c1 = compileTestClass(counter, "", "c1");
117    FileAndPath c2 = compileTestClass(counter, "", "c2");
118    packageAndLoadJar(c1, c2);
119    packageAndLoadJar(c1);
120
121    ClassFinder allClassesFinder = new ClassFinder(classLoader);
122    Set<Class<?>> allClasses = allClassesFinder.findClasses(
123        makePackageName("", counter), false);
124    assertEquals(2, allClasses.size());
125  }
126
127  @Test
128  public void testClassFinderHandlesNestedPackages() throws Exception {
129    final String NESTED = ".nested";
130    final String CLASSNAME1 = name.getMethodName() + "1";
131    final String CLASSNAME2 = name.getMethodName() + "2";
132    long counter = testCounter.incrementAndGet();
133    FileAndPath c1 = compileTestClass(counter, "", "c1");
134    FileAndPath c2 = compileTestClass(counter, NESTED, CLASSNAME1);
135    FileAndPath c3 = compileTestClass(counter, NESTED, CLASSNAME2);
136    packageAndLoadJar(c1, c2);
137    packageAndLoadJar(c3);
138
139    ClassFinder allClassesFinder = new ClassFinder(classLoader);
140    Set<Class<?>> nestedClasses = allClassesFinder.findClasses(
141        makePackageName(NESTED, counter), false);
142    assertEquals(2, nestedClasses.size());
143    Class<?> nestedClass1 = makeClass(NESTED, CLASSNAME1, counter);
144    assertTrue(nestedClasses.contains(nestedClass1));
145    Class<?> nestedClass2 = makeClass(NESTED, CLASSNAME2, counter);
146    assertTrue(nestedClasses.contains(nestedClass2));
147  }
148
149  @Test
150  public void testClassFinderFiltersByNameInJar() throws Exception {
151    final long counter = testCounter.incrementAndGet();
152    final String classNamePrefix = name.getMethodName();
153    LOG.info("Created jar " + createAndLoadJar("", classNamePrefix, counter));
154
155    ClassFinder.FileNameFilter notExcNameFilter =
156      (fileName, absFilePath) -> !fileName.startsWith(PREFIX);
157    ClassFinder incClassesFinder = new ClassFinder(null, notExcNameFilter, null, classLoader);
158    Set<Class<?>> incClasses = incClassesFinder.findClasses(
159        makePackageName("", counter), false);
160    assertEquals(1, incClasses.size());
161    Class<?> incClass = makeClass("", classNamePrefix, counter);
162    assertTrue(incClasses.contains(incClass));
163  }
164
165  @Test
166  public void testClassFinderFiltersByClassInJar() throws Exception {
167    final long counter = testCounter.incrementAndGet();
168    final String classNamePrefix = name.getMethodName();
169    LOG.info("Created jar " + createAndLoadJar("", classNamePrefix, counter));
170
171    final ClassFinder.ClassFilter notExcClassFilter = c -> !c.getSimpleName().startsWith(PREFIX);
172    ClassFinder incClassesFinder = new ClassFinder(null, null, notExcClassFilter, classLoader);
173    Set<Class<?>> incClasses = incClassesFinder.findClasses(
174        makePackageName("", counter), false);
175    assertEquals(1, incClasses.size());
176    Class<?> incClass = makeClass("", classNamePrefix, counter);
177    assertTrue(incClasses.contains(incClass));
178  }
179
180  private static String createAndLoadJar(final String packageNameSuffix,
181      final String classNamePrefix, final long counter) throws Exception {
182    FileAndPath c1 = compileTestClass(counter, packageNameSuffix, classNamePrefix);
183    FileAndPath c2 = compileTestClass(counter, packageNameSuffix, PREFIX + "1");
184    FileAndPath c3 = compileTestClass(counter, packageNameSuffix, PREFIX + classNamePrefix + "2");
185    return packageAndLoadJar(c1, c2, c3);
186  }
187
188  @Test
189  public void testClassFinderFiltersByPathInJar() throws Exception {
190    final String CLASSNAME = name.getMethodName();
191    long counter = testCounter.incrementAndGet();
192    FileAndPath c1 = compileTestClass(counter, "", CLASSNAME);
193    FileAndPath c2 = compileTestClass(counter, "", "c2");
194    packageAndLoadJar(c1);
195    final String excludedJar = packageAndLoadJar(c2);
196    /* ResourcePathFilter will pass us the resourcePath as a path of a
197     * URL from the classloader. For Windows, the ablosute path and the
198     * one from the URL have different file separators.
199     */
200    final String excludedJarResource =
201      new File(excludedJar).toURI().getRawSchemeSpecificPart();
202
203    final ClassFinder.ResourcePathFilter notExcJarFilter =
204      (resourcePath, isJar) -> !isJar || !resourcePath.equals(excludedJarResource);
205    ClassFinder incClassesFinder = new ClassFinder(notExcJarFilter, null, null, classLoader);
206    Set<Class<?>> incClasses = incClassesFinder.findClasses(
207        makePackageName("", counter), false);
208    assertEquals(1, incClasses.size());
209    Class<?> incClass = makeClass("", CLASSNAME, counter);
210    assertTrue(incClasses.contains(incClass));
211  }
212
213  @Test
214  public void testClassFinderCanFindClassesInDirs() throws Exception {
215    // Make some classes for us to find.  Class naming and packaging is kinda cryptic.
216    // TODO: Fix.
217    final long counter = testCounter.incrementAndGet();
218    final String classNamePrefix = name.getMethodName();
219    String pkgNameSuffix = name.getMethodName();
220    LOG.info("Created jar " + createAndLoadJar(pkgNameSuffix, classNamePrefix, counter));
221    ClassFinder allClassesFinder = new ClassFinder(classLoader);
222    String pkgName = makePackageName(pkgNameSuffix, counter);
223    Set<Class<?>> allClasses = allClassesFinder.findClasses(pkgName, false);
224    assertTrue("Classes in " + pkgName, allClasses.size() > 0);
225    String classNameToFind = classNamePrefix + counter;
226    assertTrue(contains(allClasses, classNameToFind));
227  }
228
229  private static boolean contains(final Set<Class<?>> classes, final String simpleName) {
230    for (Class<?> c: classes) {
231      if (c.getSimpleName().equals(simpleName)) {
232        return true;
233      }
234    }
235    return false;
236  }
237
238  @Test
239  public void testClassFinderFiltersByNameInDirs() throws Exception {
240    // Make some classes for us to find.  Class naming and packaging is kinda cryptic.
241    // TODO: Fix.
242    final long counter = testCounter.incrementAndGet();
243    final String classNamePrefix = name.getMethodName();
244    String pkgNameSuffix = name.getMethodName();
245    LOG.info("Created jar " + createAndLoadJar(pkgNameSuffix, classNamePrefix, counter));
246    final String classNameToFilterOut = classNamePrefix + counter;
247    final ClassFinder.FileNameFilter notThisFilter =
248      (fileName, absFilePath) -> !fileName.equals(classNameToFilterOut + ".class");
249    String pkgName = makePackageName(pkgNameSuffix, counter);
250    ClassFinder allClassesFinder = new ClassFinder(classLoader);
251    Set<Class<?>> allClasses = allClassesFinder.findClasses(pkgName, false);
252    assertTrue("Classes in " + pkgName, allClasses.size() > 0);
253    ClassFinder notThisClassFinder = new ClassFinder(null, notThisFilter, null, classLoader);
254    Set<Class<?>> notAllClasses = notThisClassFinder.findClasses(pkgName, false);
255    assertFalse(contains(notAllClasses, classNameToFilterOut));
256    assertEquals(allClasses.size() - 1, notAllClasses.size());
257  }
258
259  @Test
260  public void testClassFinderFiltersByClassInDirs() throws Exception {
261    // Make some classes for us to find.  Class naming and packaging is kinda cryptic.
262    // TODO: Fix.
263    final long counter = testCounter.incrementAndGet();
264    final String classNamePrefix = name.getMethodName();
265    String pkgNameSuffix = name.getMethodName();
266    LOG.info("Created jar " + createAndLoadJar(pkgNameSuffix, classNamePrefix, counter));
267    final Class<?> clazz = makeClass(pkgNameSuffix, classNamePrefix, counter);
268    final ClassFinder.ClassFilter notThisFilter = c -> c != clazz;
269    String pkgName = makePackageName(pkgNameSuffix, counter);
270    ClassFinder allClassesFinder = new ClassFinder(classLoader);
271    Set<Class<?>> allClasses = allClassesFinder.findClasses(pkgName, false);
272    assertTrue("Classes in " + pkgName, allClasses.size() > 0);
273    ClassFinder notThisClassFinder = new ClassFinder(null, null, notThisFilter, classLoader);
274    Set<Class<?>> notAllClasses = notThisClassFinder.findClasses(pkgName, false);
275    assertFalse(contains(notAllClasses, clazz.getSimpleName()));
276    assertEquals(allClasses.size() - 1, notAllClasses.size());
277  }
278
279  @Test
280  public void testClassFinderFiltersByPathInDirs() throws Exception {
281    final String hardcodedThisSubdir = "hbase-common";
282    final ClassFinder.ResourcePathFilter notExcJarFilter =
283      (resourcePath, isJar) -> isJar || !resourcePath.contains(hardcodedThisSubdir);
284    String thisPackage = this.getClass().getPackage().getName();
285    ClassFinder notThisClassFinder = new ClassFinder(notExcJarFilter, null, null, classLoader);
286    Set<Class<?>> notAllClasses = notThisClassFinder.findClasses(thisPackage, false);
287    assertFalse(notAllClasses.contains(this.getClass()));
288  }
289
290  @Test
291  public void testClassFinderDefaultsToOwnPackage() throws Exception {
292    // Correct handling of nested packages is tested elsewhere, so here we just assume
293    // pkgClasses is the correct answer that we don't have to check.
294    ClassFinder allClassesFinder = new ClassFinder(classLoader);
295    Set<Class<?>> pkgClasses = allClassesFinder.findClasses(
296        ClassFinder.class.getPackage().getName(), false);
297    Set<Class<?>> defaultClasses = allClassesFinder.findClasses(false);
298    assertArrayEquals(pkgClasses.toArray(), defaultClasses.toArray());
299  }
300
301  private static class FileAndPath {
302    String path;
303    File file;
304    public FileAndPath(String path, File file) {
305      this.file = file;
306      this.path = path;
307    }
308  }
309
310  private static Class<?> makeClass(String nestedPkgSuffix,
311      String className, long counter) throws ClassNotFoundException {
312    String name = makePackageName(nestedPkgSuffix, counter) + "." + className + counter;
313    return Class.forName(name, true, classLoader);
314  }
315
316  private static String makePackageName(String nestedSuffix, long counter) {
317    return BASEPKG + counter + nestedSuffix;
318  }
319
320  /**
321   * Compiles the test class with bogus code into a .class file.
322   * Unfortunately it's very tedious.
323   * @param counter Unique test counter.
324   * @param packageNameSuffix Package name suffix (e.g. ".suffix") for nesting, or "".
325   * @return The resulting .class file and the location in jar it is supposed to go to.
326   */
327  private static FileAndPath compileTestClass(long counter,
328      String packageNameSuffix, String classNamePrefix) throws Exception {
329    classNamePrefix = classNamePrefix + counter;
330    String packageName = makePackageName(packageNameSuffix, counter);
331    String javaPath = basePath + classNamePrefix + ".java";
332    String classPath = basePath + classNamePrefix + ".class";
333    PrintStream source = new PrintStream(javaPath);
334    source.println("package " + packageName + ";");
335    source.println("public class " + classNamePrefix
336        + " { public static void main(String[] args) { } };");
337    source.close();
338    JavaCompiler jc = ToolProvider.getSystemJavaCompiler();
339    int result = jc.run(null, null, null, javaPath);
340    assertEquals(0, result);
341    File classFile = new File(classPath);
342    assertTrue(classFile.exists());
343    return new FileAndPath(packageName.replace('.', '/') + '/', classFile);
344  }
345
346  /**
347   * Makes a jar out of some class files. Unfortunately it's very tedious.
348   * @param filesInJar Files created via compileTestClass.
349   * @return path to the resulting jar file.
350   */
351  private static String packageAndLoadJar(FileAndPath... filesInJar) throws Exception {
352    // First, write the bogus jar file.
353    String path = basePath + "jar" + jarCounter.incrementAndGet() + ".jar";
354    Manifest manifest = new Manifest();
355    manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
356    FileOutputStream fos = new FileOutputStream(path);
357    JarOutputStream jarOutputStream = new JarOutputStream(fos, manifest);
358    // Directory entries for all packages have to be added explicitly for
359    // resources to be findable via ClassLoader. Directory entries must end
360    // with "/"; the initial one is expected to, also.
361    Set<String> pathsInJar = new HashSet<>();
362    for (FileAndPath fileAndPath : filesInJar) {
363      String pathToAdd = fileAndPath.path;
364      while (pathsInJar.add(pathToAdd)) {
365        int ix = pathToAdd.lastIndexOf('/', pathToAdd.length() - 2);
366        if (ix < 0) {
367          break;
368        }
369        pathToAdd = pathToAdd.substring(0, ix);
370      }
371    }
372    for (String pathInJar : pathsInJar) {
373      jarOutputStream.putNextEntry(new JarEntry(pathInJar));
374      jarOutputStream.closeEntry();
375    }
376    for (FileAndPath fileAndPath : filesInJar) {
377      File file = fileAndPath.file;
378      jarOutputStream.putNextEntry(
379          new JarEntry(fileAndPath.path + file.getName()));
380      byte[] allBytes = new byte[(int)file.length()];
381      FileInputStream fis = new FileInputStream(file);
382      fis.read(allBytes);
383      fis.close();
384      jarOutputStream.write(allBytes);
385      jarOutputStream.closeEntry();
386    }
387    jarOutputStream.close();
388    fos.close();
389
390    // Add the file to classpath.
391    File jarFile = new File(path);
392    assertTrue(jarFile.exists());
393    classLoader.addURL(jarFile.toURI().toURL());
394    return jarFile.getAbsolutePath();
395  }
396
397  // Java 11 workaround - Custom class loader to expose addUrl method of URLClassLoader
398  private static class CustomClassloader extends URLClassLoader {
399    public CustomClassloader(URL[] urls, ClassLoader parentLoader) {
400      super(urls, parentLoader);
401    }
402
403    public void addURL(URL url) {
404      super.addURL(url);
405    }
406  }
407}