001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import java.io.File;
021import java.io.IOException;
022import java.net.ServerSocket;
023import java.util.Arrays;
024import java.util.HashSet;
025import java.util.List;
026import java.util.Set;
027import java.util.UUID;
028import java.util.concurrent.ThreadLocalRandom;
029import org.apache.commons.io.FileUtils;
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.Waiter.Predicate;
033import org.apache.hadoop.hbase.io.compress.Compression;
034import org.apache.yetus.audience.InterfaceAudience;
035import org.slf4j.Logger;
036import org.slf4j.LoggerFactory;
037
038/**
039 * Common helpers for testing HBase that do not depend on specific server/etc. things.
040 * @see org.apache.hadoop.hbase.HBaseCommonTestingUtil
041 * @deprecated since 3.0.0, will be removed in 4.0.0. Use
042 *             {@link org.apache.hadoop.hbase.testing.TestingHBaseCluster} instead.
043 */
044@InterfaceAudience.Public
045@Deprecated
046public class HBaseCommonTestingUtility {
047  protected static final Logger LOG = LoggerFactory.getLogger(HBaseCommonTestingUtility.class);
048
049  /**
050   * Compression algorithms to use in parameterized JUnit 4 tests
051   */
052  public static final List<Object[]> COMPRESSION_ALGORITHMS_PARAMETERIZED =
053    Arrays.asList(new Object[][] { { Compression.Algorithm.NONE }, { Compression.Algorithm.GZ } });
054
055  /**
056   * This is for unit tests parameterized with a two booleans.
057   */
058  public static final List<Object[]> BOOLEAN_PARAMETERIZED =
059    Arrays.asList(new Object[][] { { false }, { true } });
060
061  /**
062   * Compression algorithms to use in testing
063   */
064  public static final Compression.Algorithm[] COMPRESSION_ALGORITHMS =
065    { Compression.Algorithm.NONE, Compression.Algorithm.GZ };
066
067  protected final Configuration conf;
068
069  public HBaseCommonTestingUtility() {
070    this(null);
071  }
072
073  public HBaseCommonTestingUtility(Configuration conf) {
074    this.conf = (conf == null ? HBaseConfiguration.create() : conf);
075  }
076
077  /**
078   * Returns this classes's instance of {@link Configuration}.
079   * @return Instance of Configuration.
080   */
081  public Configuration getConfiguration() {
082    return this.conf;
083  }
084
085  /**
086   * System property key to get base test directory value
087   */
088  public static final String BASE_TEST_DIRECTORY_KEY = "test.build.data.basedirectory";
089
090  /**
091   * Default base directory for test output.
092   */
093  public static final String DEFAULT_BASE_TEST_DIRECTORY = "target/test-data";
094
095  /**
096   * Directory where we put the data for this instance of HBaseTestingUtility
097   */
098  private File dataTestDir = null;
099
100  /**
101   * @return Where to write test data on local filesystem, specific to the test. Useful for tests
102   *         that do not use a cluster. Creates it if it does not exist already.
103   */
104  public Path getDataTestDir() {
105    if (this.dataTestDir == null) {
106      setupDataTestDir();
107    }
108    return new Path(this.dataTestDir.getAbsolutePath());
109  }
110
111  /**
112   * @param name the name of a subdirectory or file in the test data directory
113   * @return Path to a subdirectory or file named {code subdirName} under {@link #getDataTestDir()}.
114   *         Does *NOT* create the directory or file if it does not exist.
115   */
116  public Path getDataTestDir(final String name) {
117    return new Path(getDataTestDir(), name);
118  }
119
120  /**
121   * Sets up a directory for a test to use.
122   * @return New directory path, if created.
123   */
124  protected Path setupDataTestDir() {
125    if (this.dataTestDir != null) {
126      LOG.warn("Data test dir already setup in " + dataTestDir.getAbsolutePath());
127      return null;
128    }
129    Path testPath = getRandomDir();
130    this.dataTestDir = new File(testPath.toString()).getAbsoluteFile();
131    // Set this property so if mapreduce jobs run, they will use this as their home dir.
132    System.setProperty("test.build.dir", this.dataTestDir.toString());
133
134    if (deleteOnExit()) {
135      this.dataTestDir.deleteOnExit();
136    }
137
138    createSubDir("hbase.local.dir", testPath, "hbase-local-dir");
139
140    return testPath;
141  }
142
143  /**
144   * @return A dir with a random (uuid) name under the test dir
145   * @see #getBaseTestDir()
146   */
147  public Path getRandomDir() {
148    return new Path(getBaseTestDir(), getRandomUUID().toString());
149  }
150
151  public static UUID getRandomUUID() {
152    return new UUID(ThreadLocalRandom.current().nextLong(), ThreadLocalRandom.current().nextLong());
153  }
154
155  protected void createSubDir(String propertyName, Path parent, String subDirName) {
156    Path newPath = new Path(parent, subDirName);
157    File newDir = new File(newPath.toString()).getAbsoluteFile();
158
159    if (deleteOnExit()) {
160      newDir.deleteOnExit();
161    }
162
163    conf.set(propertyName, newDir.getAbsolutePath());
164  }
165
166  /**
167   * @return True if we should delete testing dirs on exit.
168   */
169  boolean deleteOnExit() {
170    String v = System.getProperty("hbase.testing.preserve.testdir");
171    // Let default be true, to delete on exit.
172    return v == null ? true : !Boolean.parseBoolean(v);
173  }
174
175  /**
176   * @return True if we removed the test dirs
177   */
178  public boolean cleanupTestDir() {
179    if (deleteDir(this.dataTestDir)) {
180      this.dataTestDir = null;
181      return true;
182    }
183    return false;
184  }
185
186  /**
187   * @param subdir Test subdir name.
188   * @return True if we removed the test dir
189   */
190  public boolean cleanupTestDir(final String subdir) {
191    if (this.dataTestDir == null) {
192      return false;
193    }
194    return deleteDir(new File(this.dataTestDir, subdir));
195  }
196
197  /**
198   * @return Where to write test data on local filesystem; usually
199   *         {@link #DEFAULT_BASE_TEST_DIRECTORY} Should not be used by the unit tests, hence its's
200   *         private. Unit test will use a subdirectory of this directory.
201   * @see #setupDataTestDir()
202   */
203  private Path getBaseTestDir() {
204    String PathName = System.getProperty(BASE_TEST_DIRECTORY_KEY, DEFAULT_BASE_TEST_DIRECTORY);
205
206    return new Path(PathName);
207  }
208
209  /**
210   * @param dir Directory to delete
211   * @return True if we deleted it.
212   */
213  boolean deleteDir(final File dir) {
214    if (dir == null || !dir.exists()) {
215      return true;
216    }
217    int ntries = 0;
218    do {
219      ntries += 1;
220      try {
221        if (deleteOnExit()) {
222          FileUtils.deleteDirectory(dir);
223        }
224
225        return true;
226      } catch (IOException ex) {
227        LOG.warn("Failed to delete " + dir.getAbsolutePath());
228      } catch (IllegalArgumentException ex) {
229        LOG.warn("Failed to delete " + dir.getAbsolutePath(), ex);
230      }
231    } while (ntries < 30);
232
233    return false;
234  }
235
236  /**
237   * Wrapper method for {@link Waiter#waitFor(Configuration, long, Predicate)}.
238   */
239  public <E extends Exception> long waitFor(long timeout, Predicate<E> predicate) throws E {
240    return Waiter.waitFor(this.conf, timeout, predicate);
241  }
242
243  /**
244   * Wrapper method for {@link Waiter#waitFor(Configuration, long, long, Predicate)}.
245   */
246  public <E extends Exception> long waitFor(long timeout, long interval, Predicate<E> predicate)
247    throws E {
248    return Waiter.waitFor(this.conf, timeout, interval, predicate);
249  }
250
251  /**
252   * Wrapper method for {@link Waiter#waitFor(Configuration, long, long, boolean, Predicate)}.
253   */
254  public <E extends Exception> long waitFor(long timeout, long interval, boolean failIfTimeout,
255    Predicate<E> predicate) throws E {
256    return Waiter.waitFor(this.conf, timeout, interval, failIfTimeout, predicate);
257  }
258
259  private static final PortAllocator portAllocator = new PortAllocator();
260
261  public static int randomFreePort() {
262    return portAllocator.randomFreePort();
263  }
264
265  static class PortAllocator {
266    private static final int MIN_RANDOM_PORT = 0xc000;
267    private static final int MAX_RANDOM_PORT = 0xfffe;
268
269    /** A set of ports that have been claimed using {@link #randomFreePort()}. */
270    private final Set<Integer> takenRandomPorts = new HashSet<>();
271
272    private final AvailablePortChecker portChecker;
273
274    public PortAllocator() {
275      this.portChecker = new AvailablePortChecker() {
276        @Override
277        public boolean available(int port) {
278          try {
279            ServerSocket sock = new ServerSocket(port);
280            sock.close();
281            return true;
282          } catch (IOException ex) {
283            return false;
284          }
285        }
286      };
287    }
288
289    public PortAllocator(AvailablePortChecker portChecker) {
290      this.portChecker = portChecker;
291    }
292
293    /**
294     * Returns a random free port and marks that port as taken. Not thread-safe. Expected to be
295     * called from single-threaded test setup code/
296     */
297    public int randomFreePort() {
298      int port = 0;
299      do {
300        port = randomPort();
301        if (takenRandomPorts.contains(port)) {
302          port = 0;
303          continue;
304        }
305        takenRandomPorts.add(port);
306
307        if (!portChecker.available(port)) {
308          port = 0;
309        }
310      } while (port == 0);
311      return port;
312    }
313
314    /**
315     * Returns a random port. These ports cannot be registered with IANA and are intended for
316     * dynamic allocation (see http://bit.ly/dynports).
317     */
318    private int randomPort() {
319      return MIN_RANDOM_PORT
320        + ThreadLocalRandom.current().nextInt(MAX_RANDOM_PORT - MIN_RANDOM_PORT);
321    }
322
323    interface AvailablePortChecker {
324      boolean available(int port);
325    }
326  }
327}