001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.hamcrest.MatcherAssert.assertThat;
021import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertFalse;
024import static org.junit.Assert.assertTrue;
025
026import java.io.File;
027import java.io.FileInputStream;
028import java.io.FileOutputStream;
029import java.io.IOException;
030import java.io.PrintStream;
031import java.net.URL;
032import java.net.URLClassLoader;
033import java.util.HashSet;
034import java.util.Set;
035import java.util.concurrent.atomic.AtomicLong;
036import java.util.jar.Attributes;
037import java.util.jar.JarEntry;
038import java.util.jar.JarOutputStream;
039import java.util.jar.Manifest;
040import javax.tools.JavaCompiler;
041import javax.tools.ToolProvider;
042import org.apache.hadoop.hbase.testclassification.MiscTests;
043import org.apache.hadoop.hbase.testclassification.SmallTests;
044import org.junit.AfterClass;
045import org.junit.BeforeClass;
046import org.junit.ClassRule;
047import org.junit.Rule;
048import org.junit.Test;
049import org.junit.experimental.categories.Category;
050import org.junit.rules.TestName;
051import org.slf4j.Logger;
052import org.slf4j.LoggerFactory;
053
054@Category({ MiscTests.class, SmallTests.class })
055public class TestClassFinder {
056  @ClassRule
057  public static final HBaseClassTestRule CLASS_RULE =
058    HBaseClassTestRule.forClass(TestClassFinder.class);
059
060  private static final Logger LOG = LoggerFactory.getLogger(TestClassFinder.class);
061
062  @Rule
063  public TestName name = new TestName();
064  private static final HBaseCommonTestingUtility testUtil = new HBaseCommonTestingUtility();
065  private static final String BASEPKG = "tfcpkg";
066  private static final String PREFIX = "Prefix";
067
068  // Use unique jar/class/package names in each test case with the help
069  // of these global counters; we are mucking with ClassLoader in this test
070  // and we don't want individual test cases to conflict via it.
071  private static AtomicLong testCounter = new AtomicLong(0);
072  private static AtomicLong jarCounter = new AtomicLong(0);
073
074  private static String basePath = null;
075
076  private static CustomClassloader classLoader;
077
078  @BeforeClass
079  public static void createTestDir() throws IOException {
080    basePath = testUtil.getDataTestDir(TestClassFinder.class.getSimpleName()).toString();
081    if (!basePath.endsWith("/")) {
082      basePath += "/";
083    }
084    // Make sure we get a brand new directory.
085    File testDir = new File(basePath);
086    if (testDir.exists()) {
087      deleteTestDir();
088    }
089    assertTrue(testDir.mkdirs());
090    LOG.info("Using new, clean directory=" + testDir);
091
092    classLoader = new CustomClassloader(new URL[0], ClassLoader.getSystemClassLoader());
093  }
094
095  @AfterClass
096  public static void deleteTestDir() {
097    testUtil.cleanupTestDir(TestClassFinder.class.getSimpleName());
098  }
099
100  @Test
101  public void testClassFinderCanFindClassesInJars() throws Exception {
102    long counter = testCounter.incrementAndGet();
103    FileAndPath c1 = compileTestClass(counter, "", "c1");
104    FileAndPath c2 = compileTestClass(counter, ".nested", "c2");
105    FileAndPath c3 = compileTestClass(counter, "", "c3");
106    packageAndLoadJar(c1, c3);
107    packageAndLoadJar(c2);
108
109    ClassFinder allClassesFinder = new ClassFinder(classLoader);
110    Set<Class<?>> allClasses = allClassesFinder.findClasses(makePackageName("", counter), false);
111    assertEquals(3, allClasses.size());
112  }
113
114  @Test
115  public void testClassFinderHandlesConflicts() throws Exception {
116    long counter = testCounter.incrementAndGet();
117    FileAndPath c1 = compileTestClass(counter, "", "c1");
118    FileAndPath c2 = compileTestClass(counter, "", "c2");
119    packageAndLoadJar(c1, c2);
120    packageAndLoadJar(c1);
121
122    ClassFinder allClassesFinder = new ClassFinder(classLoader);
123    Set<Class<?>> allClasses = allClassesFinder.findClasses(makePackageName("", counter), false);
124    assertEquals(2, allClasses.size());
125  }
126
127  @Test
128  public void testClassFinderHandlesNestedPackages() throws Exception {
129    final String NESTED = ".nested";
130    final String CLASSNAME1 = name.getMethodName() + "1";
131    final String CLASSNAME2 = name.getMethodName() + "2";
132    long counter = testCounter.incrementAndGet();
133    FileAndPath c1 = compileTestClass(counter, "", "c1");
134    FileAndPath c2 = compileTestClass(counter, NESTED, CLASSNAME1);
135    FileAndPath c3 = compileTestClass(counter, NESTED, CLASSNAME2);
136    packageAndLoadJar(c1, c2);
137    packageAndLoadJar(c3);
138
139    ClassFinder allClassesFinder = new ClassFinder(classLoader);
140    Set<Class<?>> nestedClasses =
141      allClassesFinder.findClasses(makePackageName(NESTED, counter), false);
142    assertEquals(2, nestedClasses.size());
143    Class<?> nestedClass1 = makeClass(NESTED, CLASSNAME1, counter);
144    assertTrue(nestedClasses.contains(nestedClass1));
145    Class<?> nestedClass2 = makeClass(NESTED, CLASSNAME2, counter);
146    assertTrue(nestedClasses.contains(nestedClass2));
147  }
148
149  @Test
150  public void testClassFinderFiltersByNameInJar() throws Exception {
151    final long counter = testCounter.incrementAndGet();
152    final String classNamePrefix = name.getMethodName();
153    LOG.info("Created jar " + createAndLoadJar("", classNamePrefix, counter));
154
155    ClassFinder.FileNameFilter notExcNameFilter =
156      (fileName, absFilePath) -> !fileName.startsWith(PREFIX);
157    ClassFinder incClassesFinder = new ClassFinder(null, notExcNameFilter, null, classLoader);
158    Set<Class<?>> incClasses = incClassesFinder.findClasses(makePackageName("", counter), false);
159    assertEquals(1, incClasses.size());
160    Class<?> incClass = makeClass("", classNamePrefix, counter);
161    assertTrue(incClasses.contains(incClass));
162  }
163
164  @Test
165  public void testClassFinderFiltersByClassInJar() throws Exception {
166    final long counter = testCounter.incrementAndGet();
167    final String classNamePrefix = name.getMethodName();
168    LOG.info("Created jar " + createAndLoadJar("", classNamePrefix, counter));
169
170    final ClassFinder.ClassFilter notExcClassFilter = c -> !c.getSimpleName().startsWith(PREFIX);
171    ClassFinder incClassesFinder = new ClassFinder(null, null, notExcClassFilter, classLoader);
172    Set<Class<?>> incClasses = incClassesFinder.findClasses(makePackageName("", counter), false);
173    assertEquals(1, incClasses.size());
174    Class<?> incClass = makeClass("", classNamePrefix, counter);
175    assertTrue(incClasses.contains(incClass));
176  }
177
178  private static String createAndLoadJar(final String packageNameSuffix,
179    final String classNamePrefix, final long counter) throws Exception {
180    FileAndPath c1 = compileTestClass(counter, packageNameSuffix, classNamePrefix);
181    FileAndPath c2 = compileTestClass(counter, packageNameSuffix, PREFIX + "1");
182    FileAndPath c3 = compileTestClass(counter, packageNameSuffix, PREFIX + classNamePrefix + "2");
183    return packageAndLoadJar(c1, c2, c3);
184  }
185
186  @Test
187  public void testClassFinderFiltersByPathInJar() throws Exception {
188    final String CLASSNAME = name.getMethodName();
189    long counter = testCounter.incrementAndGet();
190    FileAndPath c1 = compileTestClass(counter, "", CLASSNAME);
191    FileAndPath c2 = compileTestClass(counter, "", "c2");
192    packageAndLoadJar(c1);
193    final String excludedJar = packageAndLoadJar(c2);
194    /*
195     * ResourcePathFilter will pass us the resourcePath as a path of a URL from the classloader. For
196     * Windows, the ablosute path and the one from the URL have different file separators.
197     */
198    final String excludedJarResource = new File(excludedJar).toURI().getRawSchemeSpecificPart();
199
200    final ClassFinder.ResourcePathFilter notExcJarFilter =
201      (resourcePath, isJar) -> !isJar || !resourcePath.equals(excludedJarResource);
202    ClassFinder incClassesFinder = new ClassFinder(notExcJarFilter, null, null, classLoader);
203    Set<Class<?>> incClasses = incClassesFinder.findClasses(makePackageName("", counter), false);
204    assertEquals(1, incClasses.size());
205    Class<?> incClass = makeClass("", CLASSNAME, counter);
206    assertTrue(incClasses.contains(incClass));
207  }
208
209  @Test
210  public void testClassFinderCanFindClassesInDirs() throws Exception {
211    // Make some classes for us to find. Class naming and packaging is kinda cryptic.
212    // TODO: Fix.
213    final long counter = testCounter.incrementAndGet();
214    final String classNamePrefix = name.getMethodName();
215    String pkgNameSuffix = name.getMethodName();
216    LOG.info("Created jar " + createAndLoadJar(pkgNameSuffix, classNamePrefix, counter));
217    ClassFinder allClassesFinder = new ClassFinder(classLoader);
218    String pkgName = makePackageName(pkgNameSuffix, counter);
219    Set<Class<?>> allClasses = allClassesFinder.findClasses(pkgName, false);
220    assertTrue("Classes in " + pkgName, allClasses.size() > 0);
221    String classNameToFind = classNamePrefix + counter;
222    assertTrue(contains(allClasses, classNameToFind));
223  }
224
225  private static boolean contains(final Set<Class<?>> classes, final String simpleName) {
226    for (Class<?> c : classes) {
227      if (c.getSimpleName().equals(simpleName)) {
228        return true;
229      }
230    }
231    return false;
232  }
233
234  @Test
235  public void testClassFinderFiltersByNameInDirs() throws Exception {
236    // Make some classes for us to find. Class naming and packaging is kinda cryptic.
237    // TODO: Fix.
238    final long counter = testCounter.incrementAndGet();
239    final String classNamePrefix = name.getMethodName();
240    String pkgNameSuffix = name.getMethodName();
241    LOG.info("Created jar " + createAndLoadJar(pkgNameSuffix, classNamePrefix, counter));
242    final String classNameToFilterOut = classNamePrefix + counter;
243    final ClassFinder.FileNameFilter notThisFilter =
244      (fileName, absFilePath) -> !fileName.equals(classNameToFilterOut + ".class");
245    String pkgName = makePackageName(pkgNameSuffix, counter);
246    ClassFinder allClassesFinder = new ClassFinder(classLoader);
247    Set<Class<?>> allClasses = allClassesFinder.findClasses(pkgName, false);
248    assertTrue("Classes in " + pkgName, allClasses.size() > 0);
249    ClassFinder notThisClassFinder = new ClassFinder(null, notThisFilter, null, classLoader);
250    Set<Class<?>> notAllClasses = notThisClassFinder.findClasses(pkgName, false);
251    assertFalse(contains(notAllClasses, classNameToFilterOut));
252    assertEquals(allClasses.size() - 1, notAllClasses.size());
253  }
254
255  @Test
256  public void testClassFinderFiltersByClassInDirs() throws Exception {
257    // Make some classes for us to find. Class naming and packaging is kinda cryptic.
258    // TODO: Fix.
259    final long counter = testCounter.incrementAndGet();
260    final String classNamePrefix = name.getMethodName();
261    String pkgNameSuffix = name.getMethodName();
262    LOG.info("Created jar " + createAndLoadJar(pkgNameSuffix, classNamePrefix, counter));
263    final Class<?> clazz = makeClass(pkgNameSuffix, classNamePrefix, counter);
264    final ClassFinder.ClassFilter notThisFilter = c -> c != clazz;
265    String pkgName = makePackageName(pkgNameSuffix, counter);
266    ClassFinder allClassesFinder = new ClassFinder(classLoader);
267    Set<Class<?>> allClasses = allClassesFinder.findClasses(pkgName, false);
268    assertTrue("Classes in " + pkgName, allClasses.size() > 0);
269    ClassFinder notThisClassFinder = new ClassFinder(null, null, notThisFilter, classLoader);
270    Set<Class<?>> notAllClasses = notThisClassFinder.findClasses(pkgName, false);
271    assertFalse(contains(notAllClasses, clazz.getSimpleName()));
272    assertEquals(allClasses.size() - 1, notAllClasses.size());
273  }
274
275  @Test
276  public void testClassFinderFiltersByPathInDirs() throws Exception {
277    final String hardcodedThisSubdir = "hbase-common";
278    final ClassFinder.ResourcePathFilter notExcJarFilter =
279      (resourcePath, isJar) -> isJar || !resourcePath.contains(hardcodedThisSubdir);
280    String thisPackage = this.getClass().getPackage().getName();
281    ClassFinder notThisClassFinder = new ClassFinder(notExcJarFilter, null, null, classLoader);
282    Set<Class<?>> notAllClasses = notThisClassFinder.findClasses(thisPackage, false);
283    assertFalse(notAllClasses.contains(this.getClass()));
284  }
285
286  @Test
287  public void testClassFinderDefaultsToOwnPackage() throws Exception {
288    // Correct handling of nested packages is tested elsewhere, so here we just assume
289    // pkgClasses is the correct answer that we don't have to check.
290    ClassFinder allClassesFinder = new ClassFinder(classLoader);
291    Set<Class<?>> pkgClasses =
292      allClassesFinder.findClasses(ClassFinder.class.getPackage().getName(), false);
293    Set<Class<?>> defaultClasses = allClassesFinder.findClasses(false);
294    Object[] pkgClassesArray = pkgClasses.toArray();
295    Object[] defaultClassesArray = defaultClasses.toArray();
296    assertEquals(pkgClassesArray.length, defaultClassesArray.length);
297    assertThat(pkgClassesArray, arrayContainingInAnyOrder(defaultClassesArray));
298  }
299
300  private static class FileAndPath {
301    String path;
302    File file;
303
304    public FileAndPath(String path, File file) {
305      this.file = file;
306      this.path = path;
307    }
308  }
309
310  private static Class<?> makeClass(String nestedPkgSuffix, String className, long counter)
311    throws ClassNotFoundException {
312    String name = makePackageName(nestedPkgSuffix, counter) + "." + className + counter;
313    return Class.forName(name, true, classLoader);
314  }
315
316  private static String makePackageName(String nestedSuffix, long counter) {
317    return BASEPKG + counter + nestedSuffix;
318  }
319
320  /**
321   * Compiles the test class with bogus code into a .class file. Unfortunately it's very tedious.
322   * @param counter           Unique test counter.
323   * @param packageNameSuffix Package name suffix (e.g. ".suffix") for nesting, or "".
324   * @return The resulting .class file and the location in jar it is supposed to go to.
325   */
326  private static FileAndPath compileTestClass(long counter, String packageNameSuffix,
327    String classNamePrefix) throws Exception {
328    classNamePrefix = classNamePrefix + counter;
329    String packageName = makePackageName(packageNameSuffix, counter);
330    String javaPath = basePath + classNamePrefix + ".java";
331    String classPath = basePath + classNamePrefix + ".class";
332    PrintStream source = new PrintStream(javaPath);
333    source.println("package " + packageName + ";");
334    source.println(
335      "public class " + classNamePrefix + " { public static void main(String[] args) { } };");
336    source.close();
337    JavaCompiler jc = ToolProvider.getSystemJavaCompiler();
338    int result = jc.run(null, null, null, javaPath);
339    assertEquals(0, result);
340    File classFile = new File(classPath);
341    assertTrue(classFile.exists());
342    return new FileAndPath(packageName.replace('.', '/') + '/', classFile);
343  }
344
345  /**
346   * Makes a jar out of some class files. Unfortunately it's very tedious.
347   * @param filesInJar Files created via compileTestClass.
348   * @return path to the resulting jar file.
349   */
350  private static String packageAndLoadJar(FileAndPath... filesInJar) throws Exception {
351    // First, write the bogus jar file.
352    String path = basePath + "jar" + jarCounter.incrementAndGet() + ".jar";
353    Manifest manifest = new Manifest();
354    manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
355    FileOutputStream fos = new FileOutputStream(path);
356    JarOutputStream jarOutputStream = new JarOutputStream(fos, manifest);
357    // Directory entries for all packages have to be added explicitly for
358    // resources to be findable via ClassLoader. Directory entries must end
359    // with "/"; the initial one is expected to, also.
360    Set<String> pathsInJar = new HashSet<>();
361    for (FileAndPath fileAndPath : filesInJar) {
362      String pathToAdd = fileAndPath.path;
363      while (pathsInJar.add(pathToAdd)) {
364        int ix = pathToAdd.lastIndexOf('/', pathToAdd.length() - 2);
365        if (ix < 0) {
366          break;
367        }
368        pathToAdd = pathToAdd.substring(0, ix);
369      }
370    }
371    for (String pathInJar : pathsInJar) {
372      jarOutputStream.putNextEntry(new JarEntry(pathInJar));
373      jarOutputStream.closeEntry();
374    }
375    for (FileAndPath fileAndPath : filesInJar) {
376      File file = fileAndPath.file;
377      jarOutputStream.putNextEntry(new JarEntry(fileAndPath.path + file.getName()));
378      byte[] allBytes = new byte[(int) file.length()];
379      FileInputStream fis = new FileInputStream(file);
380      fis.read(allBytes);
381      fis.close();
382      jarOutputStream.write(allBytes);
383      jarOutputStream.closeEntry();
384    }
385    jarOutputStream.close();
386    fos.close();
387
388    // Add the file to classpath.
389    File jarFile = new File(path);
390    assertTrue(jarFile.exists());
391    classLoader.addURL(jarFile.toURI().toURL());
392    return jarFile.getAbsolutePath();
393  }
394
395  // Java 11 workaround - Custom class loader to expose addUrl method of URLClassLoader
396  private static class CustomClassloader extends URLClassLoader {
397    public CustomClassloader(URL[] urls, ClassLoader parentLoader) {
398      super(urls, parentLoader);
399    }
400
401    @Override
402    public void addURL(URL url) {
403      super.addURL(url);
404    }
405  }
406}