001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import java.io.File;
021import java.io.IOException;
022import java.net.ServerSocket;
023import java.util.Arrays;
024import java.util.HashSet;
025import java.util.List;
026import java.util.Random;
027import java.util.Set;
028import java.util.UUID;
029import java.util.concurrent.ThreadLocalRandom;
030import org.apache.commons.io.FileUtils;
031import org.apache.hadoop.conf.Configuration;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.Waiter.Predicate;
034import org.apache.hadoop.hbase.io.compress.Compression;
035import org.apache.yetus.audience.InterfaceAudience;
036import org.slf4j.Logger;
037import org.slf4j.LoggerFactory;
038
039/**
040 * Common helpers for testing HBase that do not depend on specific server/etc. things.
041 * @see org.apache.hadoop.hbase.HBaseCommonTestingUtil
042 * @deprecated since 3.0.0, will be removed in 4.0.0. Use
043 *             {@link org.apache.hadoop.hbase.testing.TestingHBaseCluster} instead.
044 */
045@InterfaceAudience.Public
046@Deprecated
047public class HBaseCommonTestingUtility {
048  protected static final Logger LOG = LoggerFactory.getLogger(HBaseCommonTestingUtility.class);
049
050  /**
051   * Compression algorithms to use in parameterized JUnit 4 tests
052   */
053  public static final List<Object[]> COMPRESSION_ALGORITHMS_PARAMETERIZED =
054    Arrays.asList(new Object[][] {
055      { Compression.Algorithm.NONE },
056      { Compression.Algorithm.GZ }
057    });
058
059  /**
060   * This is for unit tests parameterized with a two booleans.
061   */
062  public static final List<Object[]> BOOLEAN_PARAMETERIZED =
063      Arrays.asList(new Object[][] {
064          {false},
065          {true}
066      });
067
068  /**
069   * Compression algorithms to use in testing
070   */
071  public static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = {
072    Compression.Algorithm.NONE, Compression.Algorithm.GZ
073  };
074
075  protected final Configuration conf;
076
077  public HBaseCommonTestingUtility() {
078    this(null);
079  }
080
081  public HBaseCommonTestingUtility(Configuration conf) {
082    this.conf = (conf == null ? HBaseConfiguration.create() : conf);
083  }
084
085  /**
086   * Returns this classes's instance of {@link Configuration}.
087   *
088   * @return Instance of Configuration.
089   */
090  public Configuration getConfiguration() {
091    return this.conf;
092  }
093
094  /**
095   * System property key to get base test directory value
096   */
097  public static final String BASE_TEST_DIRECTORY_KEY =
098      "test.build.data.basedirectory";
099
100  /**
101   * Default base directory for test output.
102   */
103  public static final String DEFAULT_BASE_TEST_DIRECTORY = "target/test-data";
104
105  /**
106   * Directory where we put the data for this instance of HBaseTestingUtility
107   */
108  private File dataTestDir = null;
109
110  /**
111   * @return Where to write test data on local filesystem, specific to the test. Useful for tests
112   *    that do not use a cluster. Creates it if it does not exist already.
113   */
114  public Path getDataTestDir() {
115    if (this.dataTestDir == null) {
116      setupDataTestDir();
117    }
118    return new Path(this.dataTestDir.getAbsolutePath());
119  }
120
121  /**
122   * @param name the name of a subdirectory or file in the test data directory
123   * @return Path to a subdirectory or file named {code subdirName} under
124   *  {@link #getDataTestDir()}. Does *NOT* create the directory or file if it does not exist.
125   */
126  public Path getDataTestDir(final String name) {
127    return new Path(getDataTestDir(), name);
128  }
129
130  /**
131   * Sets up a directory for a test to use.
132   *
133   * @return New directory path, if created.
134   */
135  protected Path setupDataTestDir() {
136    if (this.dataTestDir != null) {
137      LOG.warn("Data test dir already setup in " +
138          dataTestDir.getAbsolutePath());
139      return null;
140    }
141    Path testPath = getRandomDir();
142    this.dataTestDir = new File(testPath.toString()).getAbsoluteFile();
143    // Set this property so if mapreduce jobs run, they will use this as their home dir.
144    System.setProperty("test.build.dir", this.dataTestDir.toString());
145
146    if (deleteOnExit()) {
147      this.dataTestDir.deleteOnExit();
148    }
149
150    createSubDir("hbase.local.dir", testPath, "hbase-local-dir");
151
152    return testPath;
153  }
154
155  /**
156   * @return A dir with a random (uuid) name under the test dir
157   * @see #getBaseTestDir()
158   */
159  public Path getRandomDir() {
160    return new Path(getBaseTestDir(), getRandomUUID().toString());
161  }
162
163  public static UUID getRandomUUID() {
164    return new UUID(ThreadLocalRandom.current().nextLong(),
165                    ThreadLocalRandom.current().nextLong());
166  }
167
168  protected void createSubDir(String propertyName, Path parent, String subDirName) {
169    Path newPath = new Path(parent, subDirName);
170    File newDir = new File(newPath.toString()).getAbsoluteFile();
171
172    if (deleteOnExit()) {
173      newDir.deleteOnExit();
174    }
175
176    conf.set(propertyName, newDir.getAbsolutePath());
177  }
178
179  /**
180   * @return True if we should delete testing dirs on exit.
181   */
182  boolean deleteOnExit() {
183    String v = System.getProperty("hbase.testing.preserve.testdir");
184    // Let default be true, to delete on exit.
185    return v == null ? true : !Boolean.parseBoolean(v);
186  }
187
188  /**
189   * @return True if we removed the test dirs
190   */
191  public boolean cleanupTestDir() {
192    if (deleteDir(this.dataTestDir)) {
193      this.dataTestDir = null;
194      return true;
195    }
196    return false;
197  }
198
199  /**
200   * @param subdir Test subdir name.
201   * @return True if we removed the test dir
202   */
203  public boolean cleanupTestDir(final String subdir) {
204    if (this.dataTestDir == null) {
205      return false;
206    }
207    return deleteDir(new File(this.dataTestDir, subdir));
208  }
209
210  /**
211   * @return Where to write test data on local filesystem; usually
212   *    {@link #DEFAULT_BASE_TEST_DIRECTORY}
213   *    Should not be used by the unit tests, hence its's private.
214   *    Unit test will use a subdirectory of this directory.
215   * @see #setupDataTestDir()
216   */
217  private Path getBaseTestDir() {
218    String PathName = System.getProperty(
219        BASE_TEST_DIRECTORY_KEY, DEFAULT_BASE_TEST_DIRECTORY);
220
221    return new Path(PathName);
222  }
223
224  /**
225   * @param dir Directory to delete
226   * @return True if we deleted it.
227   */
228  boolean deleteDir(final File dir) {
229    if (dir == null || !dir.exists()) {
230      return true;
231    }
232    int ntries = 0;
233    do {
234      ntries += 1;
235      try {
236        if (deleteOnExit()) {
237          FileUtils.deleteDirectory(dir);
238        }
239
240        return true;
241      } catch (IOException ex) {
242        LOG.warn("Failed to delete " + dir.getAbsolutePath());
243      } catch (IllegalArgumentException ex) {
244        LOG.warn("Failed to delete " + dir.getAbsolutePath(), ex);
245      }
246    } while (ntries < 30);
247
248    return false;
249  }
250
251  /**
252   * Wrapper method for {@link Waiter#waitFor(Configuration, long, Predicate)}.
253   */
254  public <E extends Exception> long waitFor(long timeout, Predicate<E> predicate)
255      throws E {
256    return Waiter.waitFor(this.conf, timeout, predicate);
257  }
258
259  /**
260   * Wrapper method for {@link Waiter#waitFor(Configuration, long, long, Predicate)}.
261   */
262  public <E extends Exception> long waitFor(long timeout, long interval, Predicate<E> predicate)
263      throws E {
264    return Waiter.waitFor(this.conf, timeout, interval, predicate);
265  }
266
267  /**
268   * Wrapper method for {@link Waiter#waitFor(Configuration, long, long, boolean, Predicate)}.
269   */
270  public <E extends Exception> long waitFor(long timeout, long interval,
271      boolean failIfTimeout, Predicate<E> predicate) throws E {
272    return Waiter.waitFor(this.conf, timeout, interval, failIfTimeout, predicate);
273  }
274
275  // Support for Random Port Generation.
276  static Random random = new Random();
277
278  private static final PortAllocator portAllocator = new PortAllocator(random);
279
280  public static int randomFreePort() {
281    return portAllocator.randomFreePort();
282  }
283
284  static class PortAllocator {
285    private static final int MIN_RANDOM_PORT = 0xc000;
286    private static final int MAX_RANDOM_PORT = 0xfffe;
287
288    /** A set of ports that have been claimed using {@link #randomFreePort()}. */
289    private final Set<Integer> takenRandomPorts = new HashSet<>();
290
291    private final Random random;
292    private final AvailablePortChecker portChecker;
293
294    public PortAllocator(Random random) {
295      this.random = random;
296      this.portChecker = new AvailablePortChecker() {
297        @Override
298        public boolean available(int port) {
299          try {
300            ServerSocket sock = new ServerSocket(port);
301            sock.close();
302            return true;
303          } catch (IOException ex) {
304            return false;
305          }
306        }
307      };
308    }
309
310    public PortAllocator(Random random, AvailablePortChecker portChecker) {
311      this.random = random;
312      this.portChecker = portChecker;
313    }
314
315    /**
316     * Returns a random free port and marks that port as taken. Not thread-safe. Expected to be
317     * called from single-threaded test setup code/
318     */
319    public int randomFreePort() {
320      int port = 0;
321      do {
322        port = randomPort();
323        if (takenRandomPorts.contains(port)) {
324          port = 0;
325          continue;
326        }
327        takenRandomPorts.add(port);
328
329        if (!portChecker.available(port)) {
330          port = 0;
331        }
332      } while (port == 0);
333      return port;
334    }
335
336    /**
337     * Returns a random port. These ports cannot be registered with IANA and are
338     * intended for dynamic allocation (see http://bit.ly/dynports).
339     */
340    private int randomPort() {
341      return MIN_RANDOM_PORT
342        + random.nextInt(MAX_RANDOM_PORT - MIN_RANDOM_PORT);
343    }
344
345    interface AvailablePortChecker {
346      boolean available(int port);
347    }
348  }
349}