001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import java.io.File;
021import java.io.IOException;
022import java.net.ServerSocket;
023import java.util.Arrays;
024import java.util.HashSet;
025import java.util.List;
026import java.util.Random;
027import java.util.Set;
028import java.util.UUID;
029import java.util.concurrent.ThreadLocalRandom;
030import org.apache.commons.io.FileUtils;
031import org.apache.hadoop.conf.Configuration;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.Waiter.Predicate;
034import org.apache.hadoop.hbase.io.compress.Compression;
035import org.apache.yetus.audience.InterfaceAudience;
036import org.slf4j.Logger;
037import org.slf4j.LoggerFactory;
038
039/**
040 * Common helpers for testing HBase that do not depend on specific server/etc. things.
041 * @see org.apache.hadoop.hbase.HBaseCommonTestingUtility
042 */
043@InterfaceAudience.Public
044public class HBaseCommonTestingUtility {
045  protected static final Logger LOG = LoggerFactory.getLogger(HBaseCommonTestingUtility.class);
046
047  /**
048   * Compression algorithms to use in parameterized JUnit 4 tests
049   */
050  public static final List<Object[]> COMPRESSION_ALGORITHMS_PARAMETERIZED =
051    Arrays.asList(new Object[][] {
052      { Compression.Algorithm.NONE },
053      { Compression.Algorithm.GZ }
054    });
055
056  /**
057   * This is for unit tests parameterized with a two booleans.
058   */
059  public static final List<Object[]> BOOLEAN_PARAMETERIZED =
060      Arrays.asList(new Object[][] {
061          {false},
062          {true}
063      });
064
065  /**
066   * Compression algorithms to use in testing
067   */
068  public static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = {
069    Compression.Algorithm.NONE, Compression.Algorithm.GZ
070  };
071
072  protected final Configuration conf;
073
074  public HBaseCommonTestingUtility() {
075    this(null);
076  }
077
078  public HBaseCommonTestingUtility(Configuration conf) {
079    this.conf = (conf == null ? HBaseConfiguration.create() : conf);
080  }
081
082  /**
083   * Returns this classes's instance of {@link Configuration}.
084   *
085   * @return Instance of Configuration.
086   */
087  public Configuration getConfiguration() {
088    return this.conf;
089  }
090
091  /**
092   * System property key to get base test directory value
093   */
094  public static final String BASE_TEST_DIRECTORY_KEY =
095      "test.build.data.basedirectory";
096
097  /**
098   * Default base directory for test output.
099   */
100  public static final String DEFAULT_BASE_TEST_DIRECTORY = "target/test-data";
101
102  /**
103   * Directory where we put the data for this instance of HBaseTestingUtility
104   */
105  private File dataTestDir = null;
106
107  /**
108   * @return Where to write test data on local filesystem, specific to the test. Useful for tests
109   *    that do not use a cluster. Creates it if it does not exist already.
110   */
111  public Path getDataTestDir() {
112    if (this.dataTestDir == null) {
113      setupDataTestDir();
114    }
115    return new Path(this.dataTestDir.getAbsolutePath());
116  }
117
118  /**
119   * @param name the name of a subdirectory or file in the test data directory
120   * @return Path to a subdirectory or file named {code subdirName} under
121   *  {@link #getDataTestDir()}. Does *NOT* create the directory or file if it does not exist.
122   */
123  public Path getDataTestDir(final String name) {
124    return new Path(getDataTestDir(), name);
125  }
126
127  /**
128   * Sets up a directory for a test to use.
129   *
130   * @return New directory path, if created.
131   */
132  protected Path setupDataTestDir() {
133    if (this.dataTestDir != null) {
134      LOG.warn("Data test dir already setup in " +
135          dataTestDir.getAbsolutePath());
136      return null;
137    }
138    Path testPath = getRandomDir();
139    this.dataTestDir = new File(testPath.toString()).getAbsoluteFile();
140    // Set this property so if mapreduce jobs run, they will use this as their home dir.
141    System.setProperty("test.build.dir", this.dataTestDir.toString());
142
143    if (deleteOnExit()) {
144      this.dataTestDir.deleteOnExit();
145    }
146
147    createSubDir("hbase.local.dir", testPath, "hbase-local-dir");
148
149    return testPath;
150  }
151
152  /**
153   * @return A dir with a random (uuid) name under the test dir
154   * @see #getBaseTestDir()
155   */
156  public Path getRandomDir() {
157    return new Path(getBaseTestDir(), getRandomUUID().toString());
158  }
159
160  public static UUID getRandomUUID() {
161    return new UUID(ThreadLocalRandom.current().nextLong(),
162                    ThreadLocalRandom.current().nextLong());
163  }
164
165  protected void createSubDir(String propertyName, Path parent, String subDirName) {
166    Path newPath = new Path(parent, subDirName);
167    File newDir = new File(newPath.toString()).getAbsoluteFile();
168
169    if (deleteOnExit()) {
170      newDir.deleteOnExit();
171    }
172
173    conf.set(propertyName, newDir.getAbsolutePath());
174  }
175
176  /**
177   * @return True if we should delete testing dirs on exit.
178   */
179  boolean deleteOnExit() {
180    String v = System.getProperty("hbase.testing.preserve.testdir");
181    // Let default be true, to delete on exit.
182    return v == null ? true : !Boolean.parseBoolean(v);
183  }
184
185  /**
186   * @return True if we removed the test dirs
187   */
188  public boolean cleanupTestDir() {
189    if (deleteDir(this.dataTestDir)) {
190      this.dataTestDir = null;
191      return true;
192    }
193    return false;
194  }
195
196  /**
197   * @param subdir Test subdir name.
198   * @return True if we removed the test dir
199   */
200  boolean cleanupTestDir(final String subdir) {
201    if (this.dataTestDir == null) {
202      return false;
203    }
204    return deleteDir(new File(this.dataTestDir, subdir));
205  }
206
207  /**
208   * @return Where to write test data on local filesystem; usually
209   *    {@link #DEFAULT_BASE_TEST_DIRECTORY}
210   *    Should not be used by the unit tests, hence its's private.
211   *    Unit test will use a subdirectory of this directory.
212   * @see #setupDataTestDir()
213   */
214  private Path getBaseTestDir() {
215    String PathName = System.getProperty(
216        BASE_TEST_DIRECTORY_KEY, DEFAULT_BASE_TEST_DIRECTORY);
217
218    return new Path(PathName);
219  }
220
221  /**
222   * @param dir Directory to delete
223   * @return True if we deleted it.
224   */
225  boolean deleteDir(final File dir) {
226    if (dir == null || !dir.exists()) {
227      return true;
228    }
229    int ntries = 0;
230    do {
231      ntries += 1;
232      try {
233        if (deleteOnExit()) {
234          FileUtils.deleteDirectory(dir);
235        }
236
237        return true;
238      } catch (IOException ex) {
239        LOG.warn("Failed to delete " + dir.getAbsolutePath());
240      } catch (IllegalArgumentException ex) {
241        LOG.warn("Failed to delete " + dir.getAbsolutePath(), ex);
242      }
243    } while (ntries < 30);
244
245    return false;
246  }
247
248  /**
249   * Wrapper method for {@link Waiter#waitFor(Configuration, long, Predicate)}.
250   */
251  public <E extends Exception> long waitFor(long timeout, Predicate<E> predicate)
252      throws E {
253    return Waiter.waitFor(this.conf, timeout, predicate);
254  }
255
256  /**
257   * Wrapper method for {@link Waiter#waitFor(Configuration, long, long, Predicate)}.
258   */
259  public <E extends Exception> long waitFor(long timeout, long interval, Predicate<E> predicate)
260      throws E {
261    return Waiter.waitFor(this.conf, timeout, interval, predicate);
262  }
263
264  /**
265   * Wrapper method for {@link Waiter#waitFor(Configuration, long, long, boolean, Predicate)}.
266   */
267  public <E extends Exception> long waitFor(long timeout, long interval,
268      boolean failIfTimeout, Predicate<E> predicate) throws E {
269    return Waiter.waitFor(this.conf, timeout, interval, failIfTimeout, predicate);
270  }
271
272  // Support for Random Port Generation.
273  static Random random = new Random();
274
275  private static final PortAllocator portAllocator = new PortAllocator(random);
276
277  public static int randomFreePort() {
278    return portAllocator.randomFreePort();
279  }
280
281  static class PortAllocator {
282    private static final int MIN_RANDOM_PORT = 0xc000;
283    private static final int MAX_RANDOM_PORT = 0xfffe;
284
285    /** A set of ports that have been claimed using {@link #randomFreePort()}. */
286    private final Set<Integer> takenRandomPorts = new HashSet<>();
287
288    private final Random random;
289    private final AvailablePortChecker portChecker;
290
291    public PortAllocator(Random random) {
292      this.random = random;
293      this.portChecker = new AvailablePortChecker() {
294        @Override
295        public boolean available(int port) {
296          try {
297            ServerSocket sock = new ServerSocket(port);
298            sock.close();
299            return true;
300          } catch (IOException ex) {
301            return false;
302          }
303        }
304      };
305    }
306
307    public PortAllocator(Random random, AvailablePortChecker portChecker) {
308      this.random = random;
309      this.portChecker = portChecker;
310    }
311
312    /**
313     * Returns a random free port and marks that port as taken. Not thread-safe. Expected to be
314     * called from single-threaded test setup code/
315     */
316    public int randomFreePort() {
317      int port = 0;
318      do {
319        port = randomPort();
320        if (takenRandomPorts.contains(port)) {
321          port = 0;
322          continue;
323        }
324        takenRandomPorts.add(port);
325
326        if (!portChecker.available(port)) {
327          port = 0;
328        }
329      } while (port == 0);
330      return port;
331    }
332
333    /**
334     * Returns a random port. These ports cannot be registered with IANA and are
335     * intended for dynamic allocation (see http://bit.ly/dynports).
336     */
337    private int randomPort() {
338      return MIN_RANDOM_PORT
339        + random.nextInt(MAX_RANDOM_PORT - MIN_RANDOM_PORT);
340    }
341
342    interface AvailablePortChecker {
343      boolean available(int port);
344    }
345  }
346}