001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import java.io.File;
021import java.io.IOException;
022import java.net.ServerSocket;
023import java.util.Arrays;
024import java.util.HashSet;
025import java.util.List;
026import java.util.Set;
027import java.util.UUID;
028import java.util.concurrent.ThreadLocalRandom;
029import org.apache.commons.io.FileUtils;
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.Waiter.Predicate;
033import org.apache.hadoop.hbase.io.compress.Compression;
034import org.apache.yetus.audience.InterfaceAudience;
035import org.slf4j.Logger;
036import org.slf4j.LoggerFactory;
037
038/**
039 * Common helpers for testing HBase that do not depend on specific server/etc. things.
040 * @see org.apache.hadoop.hbase.HBaseCommonTestingUtil
041 * @deprecated since 3.0.0, will be removed in 4.0.0. Use
042 *             {@link org.apache.hadoop.hbase.testing.TestingHBaseCluster} instead.
043 */
044@InterfaceAudience.Public
045@Deprecated
046public class HBaseCommonTestingUtility {
047  protected static final Logger LOG = LoggerFactory.getLogger(HBaseCommonTestingUtility.class);
048
049  /**
050   * Compression algorithms to use in parameterized JUnit 4 tests
051   */
052  public static final List<Object[]> COMPRESSION_ALGORITHMS_PARAMETERIZED =
053    Arrays.asList(new Object[][] { { Compression.Algorithm.NONE }, { Compression.Algorithm.GZ } });
054
055  /**
056   * This is for unit tests parameterized with a two booleans.
057   */
058  public static final List<Object[]> BOOLEAN_PARAMETERIZED =
059    Arrays.asList(new Object[][] { { false }, { true } });
060
061  /**
062   * Compression algorithms to use in testing
063   */
064  public static final Compression.Algorithm[] COMPRESSION_ALGORITHMS =
065    { Compression.Algorithm.NONE, Compression.Algorithm.GZ };
066
067  protected final Configuration conf;
068
069  public HBaseCommonTestingUtility() {
070    this(null);
071  }
072
073  public HBaseCommonTestingUtility(Configuration conf) {
074    this.conf = (conf == null ? HBaseConfiguration.create() : conf);
075  }
076
077  /**
078   * Returns this classes's instance of {@link Configuration}.
079   * @return Instance of Configuration.
080   */
081  public Configuration getConfiguration() {
082    return this.conf;
083  }
084
085  /**
086   * System property key to get base test directory value
087   */
088  public static final String BASE_TEST_DIRECTORY_KEY = "test.build.data.basedirectory";
089
090  /**
091   * Default base directory for test output.
092   */
093  public static final String DEFAULT_BASE_TEST_DIRECTORY = "target/test-data";
094
095  /**
096   * Directory where we put the data for this instance of HBaseTestingUtility
097   */
098  private File dataTestDir = null;
099
100  /**
101   * @return Where to write test data on local filesystem, specific to the test. Useful for tests
102   *         that do not use a cluster. Creates it if it does not exist already.
103   */
104  public Path getDataTestDir() {
105    if (this.dataTestDir == null) {
106      setupDataTestDir();
107    }
108    return new Path(this.dataTestDir.getAbsolutePath());
109  }
110
111  /**
112   * @param name the name of a subdirectory or file in the test data directory
113   * @return Path to a subdirectory or file named {code subdirName} under {@link #getDataTestDir()}.
114   *         Does *NOT* create the directory or file if it does not exist.
115   */
116  public Path getDataTestDir(final String name) {
117    return new Path(getDataTestDir(), name);
118  }
119
120  /**
121   * Sets up a directory for a test to use.
122   * @return New directory path, if created.
123   */
124  protected Path setupDataTestDir() {
125    if (this.dataTestDir != null) {
126      LOG.warn("Data test dir already setup in " + dataTestDir.getAbsolutePath());
127      return null;
128    }
129    Path testPath = getRandomDir();
130    this.dataTestDir = new File(testPath.toString()).getAbsoluteFile();
131    // Set this property so if mapreduce jobs run, they will use this as their home dir.
132    System.setProperty("test.build.dir", this.dataTestDir.toString());
133
134    if (deleteOnExit()) {
135      this.dataTestDir.deleteOnExit();
136    }
137
138    createSubDir("hbase.local.dir", testPath, "hbase-local-dir");
139
140    return testPath;
141  }
142
143  /**
144   * @return A dir with a random (uuid) name under the test dir
145   * @see #getBaseTestDir()
146   */
147  public Path getRandomDir() {
148    return new Path(getBaseTestDir(), getRandomUUID().toString());
149  }
150
151  public static UUID getRandomUUID() {
152    return new UUID(ThreadLocalRandom.current().nextLong(), ThreadLocalRandom.current().nextLong());
153  }
154
155  protected void createSubDir(String propertyName, Path parent, String subDirName) {
156    Path newPath = new Path(parent, subDirName);
157    File newDir = new File(newPath.toString()).getAbsoluteFile();
158
159    if (deleteOnExit()) {
160      newDir.deleteOnExit();
161    }
162
163    conf.set(propertyName, newDir.getAbsolutePath());
164  }
165
166  /** Returns True if we should delete testing dirs on exit. */
167  boolean deleteOnExit() {
168    String v = System.getProperty("hbase.testing.preserve.testdir");
169    // Let default be true, to delete on exit.
170    return v == null ? true : !Boolean.parseBoolean(v);
171  }
172
173  /** Returns True if we removed the test dirs */
174  public boolean cleanupTestDir() {
175    if (deleteDir(this.dataTestDir)) {
176      this.dataTestDir = null;
177      return true;
178    }
179    return false;
180  }
181
182  /**
183   * @param subdir Test subdir name.
184   * @return True if we removed the test dir
185   */
186  public boolean cleanupTestDir(final String subdir) {
187    if (this.dataTestDir == null) {
188      return false;
189    }
190    return deleteDir(new File(this.dataTestDir, subdir));
191  }
192
193  /**
194   * @return Where to write test data on local filesystem; usually
195   *         {@link #DEFAULT_BASE_TEST_DIRECTORY} Should not be used by the unit tests, hence its's
196   *         private. Unit test will use a subdirectory of this directory.
197   * @see #setupDataTestDir()
198   */
199  private Path getBaseTestDir() {
200    String PathName = System.getProperty(BASE_TEST_DIRECTORY_KEY, DEFAULT_BASE_TEST_DIRECTORY);
201
202    return new Path(PathName);
203  }
204
205  /**
206   * @param dir Directory to delete
207   * @return True if we deleted it.
208   */
209  boolean deleteDir(final File dir) {
210    if (dir == null || !dir.exists()) {
211      return true;
212    }
213    int ntries = 0;
214    do {
215      ntries += 1;
216      try {
217        if (deleteOnExit()) {
218          FileUtils.deleteDirectory(dir);
219        }
220
221        return true;
222      } catch (IOException ex) {
223        LOG.warn("Failed to delete " + dir.getAbsolutePath());
224      } catch (IllegalArgumentException ex) {
225        LOG.warn("Failed to delete " + dir.getAbsolutePath(), ex);
226      }
227    } while (ntries < 30);
228
229    return false;
230  }
231
232  /**
233   * Wrapper method for {@link Waiter#waitFor(Configuration, long, Predicate)}.
234   */
235  public <E extends Exception> long waitFor(long timeout, Predicate<E> predicate) throws E {
236    return Waiter.waitFor(this.conf, timeout, predicate);
237  }
238
239  /**
240   * Wrapper method for {@link Waiter#waitFor(Configuration, long, long, Predicate)}.
241   */
242  public <E extends Exception> long waitFor(long timeout, long interval, Predicate<E> predicate)
243    throws E {
244    return Waiter.waitFor(this.conf, timeout, interval, predicate);
245  }
246
247  /**
248   * Wrapper method for {@link Waiter#waitFor(Configuration, long, long, boolean, Predicate)}.
249   */
250  public <E extends Exception> long waitFor(long timeout, long interval, boolean failIfTimeout,
251    Predicate<E> predicate) throws E {
252    return Waiter.waitFor(this.conf, timeout, interval, failIfTimeout, predicate);
253  }
254
255  private static final PortAllocator portAllocator = new PortAllocator();
256
257  public static int randomFreePort() {
258    return portAllocator.randomFreePort();
259  }
260
261  static class PortAllocator {
262    private static final int MIN_RANDOM_PORT = 0xc000;
263    private static final int MAX_RANDOM_PORT = 0xfffe;
264
265    /** A set of ports that have been claimed using {@link #randomFreePort()}. */
266    private final Set<Integer> takenRandomPorts = new HashSet<>();
267
268    private final AvailablePortChecker portChecker;
269
270    public PortAllocator() {
271      this.portChecker = new AvailablePortChecker() {
272        @Override
273        public boolean available(int port) {
274          try {
275            ServerSocket sock = new ServerSocket(port);
276            sock.close();
277            return true;
278          } catch (IOException ex) {
279            return false;
280          }
281        }
282      };
283    }
284
285    public PortAllocator(AvailablePortChecker portChecker) {
286      this.portChecker = portChecker;
287    }
288
289    /**
290     * Returns a random free port and marks that port as taken. Not thread-safe. Expected to be
291     * called from single-threaded test setup code/
292     */
293    public int randomFreePort() {
294      int port = 0;
295      do {
296        port = randomPort();
297        if (takenRandomPorts.contains(port)) {
298          port = 0;
299          continue;
300        }
301        takenRandomPorts.add(port);
302
303        if (!portChecker.available(port)) {
304          port = 0;
305        }
306      } while (port == 0);
307      return port;
308    }
309
310    /**
311     * Returns a random port. These ports cannot be registered with IANA and are intended for
312     * dynamic allocation (see http://bit.ly/dynports).
313     */
314    private int randomPort() {
315      return MIN_RANDOM_PORT
316        + ThreadLocalRandom.current().nextInt(MAX_RANDOM_PORT - MIN_RANDOM_PORT);
317    }
318
319    interface AvailablePortChecker {
320      boolean available(int port);
321    }
322  }
323}