001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import java.io.File;
021import java.io.IOException;
022import java.net.ServerSocket;
023import java.util.Arrays;
024import java.util.HashSet;
025import java.util.List;
026import java.util.Random;
027import java.util.Set;
028import java.util.UUID;
029import java.util.concurrent.ThreadLocalRandom;
030import org.apache.commons.io.FileUtils;
031import org.apache.hadoop.conf.Configuration;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.Waiter.Predicate;
034import org.apache.hadoop.hbase.io.compress.Compression;
035import org.apache.yetus.audience.InterfaceAudience;
036import org.apache.yetus.audience.InterfaceStability;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040/**
041 * Common helpers for testing HBase that do not depend on specific server/etc. things.
042 */
043@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.PHOENIX)
044@InterfaceStability.Evolving
045public class HBaseCommonTestingUtil {
046  protected static final Logger LOG = LoggerFactory.getLogger(HBaseCommonTestingUtil.class);
047
048  /**
049   * Compression algorithms to use in parameterized JUnit 4 tests
050   */
051  public static final List<Object[]> COMPRESSION_ALGORITHMS_PARAMETERIZED =
052    Arrays.asList(new Object[][] { { Compression.Algorithm.NONE }, { Compression.Algorithm.GZ } });
053
054  /**
055   * This is for unit tests parameterized with a two booleans.
056   */
057  public static final List<Object[]> BOOLEAN_PARAMETERIZED =
058    Arrays.asList(new Object[][] { { false }, { true } });
059
060  /**
061   * Compression algorithms to use in testing
062   */
063  public static final Compression.Algorithm[] COMPRESSION_ALGORITHMS =
064    { Compression.Algorithm.NONE, Compression.Algorithm.GZ };
065
066  protected final Configuration conf;
067
068  public HBaseCommonTestingUtil() {
069    this(null);
070  }
071
072  public HBaseCommonTestingUtil(Configuration conf) {
073    this.conf = (conf == null ? HBaseConfiguration.create() : conf);
074  }
075
076  /**
077   * Returns this classes's instance of {@link Configuration}.
078   * @return Instance of Configuration.
079   */
080  public Configuration getConfiguration() {
081    return this.conf;
082  }
083
084  /**
085   * System property key to get base test directory value
086   */
087  public static final String BASE_TEST_DIRECTORY_KEY = "test.build.data.basedirectory";
088
089  /**
090   * Default base directory for test output.
091   */
092  public static final String DEFAULT_BASE_TEST_DIRECTORY = "target/test-data";
093
094  /**
095   * Directory where we put the data for this instance of HBaseTestingUtility
096   */
097  private File dataTestDir = null;
098
099  /**
100   * Returns Where to write test data on local filesystem, specific to the test. Useful for tests
101   * that do not use a cluster. Creates it if it does not exist already.
102   */
103  public Path getDataTestDir() {
104    if (this.dataTestDir == null) {
105      setupDataTestDir();
106    }
107    return new Path(this.dataTestDir.getAbsolutePath());
108  }
109
110  /**
111   * Returns the path to a subdirectory or file named {code subdirName} under
112   * {@link #getDataTestDir()}. Does *NOT* create the directory or file if it does not exist.
113   * @param name the name of a subdirectory or file in the test data directory
114   */
115  public Path getDataTestDir(final String name) {
116    return new Path(getDataTestDir(), name);
117  }
118
119  /**
120   * Sets up a directory for a test to use.
121   * @return New directory path, if created.
122   */
123  protected Path setupDataTestDir() {
124    if (this.dataTestDir != null) {
125      LOG.warn("Data test dir already setup in " + dataTestDir.getAbsolutePath());
126      return null;
127    }
128    Path testPath = getRandomDir();
129    this.dataTestDir = new File(testPath.toString()).getAbsoluteFile();
130    // Set this property so if mapreduce jobs run, they will use this as their home dir.
131    System.setProperty("test.build.dir", this.dataTestDir.toString());
132
133    if (deleteOnExit()) {
134      this.dataTestDir.deleteOnExit();
135    }
136
137    createSubDir("hbase.local.dir", testPath, "hbase-local-dir");
138
139    return testPath;
140  }
141
142  /**
143   * Returns a dir with a random (uuid) name under the test dir
144   * @see #getBaseTestDir()
145   */
146  public Path getRandomDir() {
147    return new Path(getBaseTestDir(), getRandomUUID().toString());
148  }
149
150  public static UUID getRandomUUID() {
151    return new UUID(ThreadLocalRandom.current().nextLong(), ThreadLocalRandom.current().nextLong());
152  }
153
154  protected void createSubDir(String propertyName, Path parent, String subDirName) {
155    Path newPath = new Path(parent, subDirName);
156    File newDir = new File(newPath.toString()).getAbsoluteFile();
157
158    if (deleteOnExit()) {
159      newDir.deleteOnExit();
160    }
161
162    conf.set(propertyName, newDir.getAbsolutePath());
163  }
164
165  /**
166   * Returns true if we should delete testing dirs on exit.
167   */
168  boolean deleteOnExit() {
169    String v = System.getProperty("hbase.testing.preserve.testdir");
170    // Let default be true, to delete on exit.
171    return v == null ? true : !Boolean.parseBoolean(v);
172  }
173
174  /**
175   * Returns true if we removed the test dirs
176   */
177  public boolean cleanupTestDir() {
178    if (deleteDir(this.dataTestDir)) {
179      this.dataTestDir = null;
180      return true;
181    }
182    return false;
183  }
184
185  /**
186   * Returns true if we removed the test dir
187   * @param subdir Test subdir name.
188   */
189  public boolean cleanupTestDir(final String subdir) {
190    if (this.dataTestDir == null) {
191      return false;
192    }
193    return deleteDir(new File(this.dataTestDir, subdir));
194  }
195
196  /**
197   * Returns Where to write test data on local filesystem; usually
198   * {@link #DEFAULT_BASE_TEST_DIRECTORY} Should not be used by the unit tests, hence its's private.
199   * Unit test will use a subdirectory of this directory.
200   * @see #setupDataTestDir()
201   */
202  private Path getBaseTestDir() {
203    String PathName = System.getProperty(BASE_TEST_DIRECTORY_KEY, DEFAULT_BASE_TEST_DIRECTORY);
204
205    return new Path(PathName);
206  }
207
208  /**
209   * Returns true if we deleted it.
210   * @param dir Directory to delete
211   */
212  boolean deleteDir(final File dir) {
213    if (dir == null || !dir.exists()) {
214      return true;
215    }
216    int ntries = 0;
217    do {
218      ntries += 1;
219      try {
220        if (deleteOnExit()) {
221          FileUtils.deleteDirectory(dir);
222        }
223
224        return true;
225      } catch (IOException ex) {
226        LOG.warn("Failed to delete " + dir.getAbsolutePath());
227      } catch (IllegalArgumentException ex) {
228        LOG.warn("Failed to delete " + dir.getAbsolutePath(), ex);
229      }
230    } while (ntries < 30);
231
232    return false;
233  }
234
235  /**
236   * Wrapper method for {@link Waiter#waitFor(Configuration, long, Predicate)}.
237   */
238  public <E extends Exception> long waitFor(long timeout, Predicate<E> predicate) throws E {
239    return Waiter.waitFor(this.conf, timeout, predicate);
240  }
241
242  /**
243   * Wrapper method for {@link Waiter#waitFor(Configuration, long, long, Predicate)}.
244   */
245  public <E extends Exception> long waitFor(long timeout, long interval, Predicate<E> predicate)
246    throws E {
247    return Waiter.waitFor(this.conf, timeout, interval, predicate);
248  }
249
250  /**
251   * Wrapper method for {@link Waiter#waitFor(Configuration, long, long, boolean, Predicate)}.
252   */
253  public <E extends Exception> long waitFor(long timeout, long interval, boolean failIfTimeout,
254    Predicate<E> predicate) throws E {
255    return Waiter.waitFor(this.conf, timeout, interval, failIfTimeout, predicate);
256  }
257
258  private static final PortAllocator portAllocator = new PortAllocator();
259
260  public static int randomFreePort() {
261    return portAllocator.randomFreePort();
262  }
263
264  static class PortAllocator {
265    private static final int MIN_RANDOM_PORT = 0xc000;
266    private static final int MAX_RANDOM_PORT = 0xfffe;
267
268    /** A set of ports that have been claimed using {@link #randomFreePort()}. */
269    private final Set<Integer> takenRandomPorts = new HashSet<>();
270    private final Random random;
271    private final AvailablePortChecker portChecker;
272
273    public PortAllocator() {
274      this.random = new Random();
275      this.portChecker = new AvailablePortChecker() {
276        @Override
277        public boolean available(int port) {
278          try {
279            ServerSocket sock = new ServerSocket(port);
280            sock.close();
281            return true;
282          } catch (IOException ex) {
283            return false;
284          }
285        }
286      };
287    }
288
289    public PortAllocator(Random random, AvailablePortChecker portChecker) {
290      this.random = random;
291      this.portChecker = portChecker;
292    }
293
294    public PortAllocator(AvailablePortChecker portChecker) {
295      this(new Random(), portChecker);
296    }
297
298    /**
299     * Returns a random free port and marks that port as taken. Not thread-safe. Expected to be
300     * called from single-threaded test setup code/
301     */
302    public int randomFreePort() {
303      int port = 0;
304      do {
305        port = randomPort();
306        if (takenRandomPorts.contains(port)) {
307          port = 0;
308          continue;
309        }
310        takenRandomPorts.add(port);
311
312        if (!portChecker.available(port)) {
313          port = 0;
314        }
315      } while (port == 0);
316      return port;
317    }
318
319    /**
320     * Returns a random port. These ports cannot be registered with IANA and are intended for
321     * dynamic allocation (see http://bit.ly/dynports).
322     */
323    private int randomPort() {
324      return MIN_RANDOM_PORT + random.nextInt(MAX_RANDOM_PORT - MIN_RANDOM_PORT);
325    }
326
327    interface AvailablePortChecker {
328      boolean available(int port);
329    }
330  }
331}