001/**
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019package org.apache.hadoop.hbase;
020
021import java.io.IOException;
022import java.nio.charset.StandardCharsets;
023import java.util.NavigableMap;
024
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.client.Durability;
029import org.apache.hadoop.hbase.client.Get;
030import org.apache.hadoop.hbase.client.Put;
031import org.apache.hadoop.hbase.client.Result;
032import org.apache.hadoop.hbase.client.Table;
033import org.apache.hadoop.hbase.log.HBaseMarkers;
034import org.apache.hadoop.hbase.regionserver.HRegion;
035import org.apache.hadoop.hbase.regionserver.Region;
036import org.apache.hadoop.hbase.regionserver.RegionAsTable;
037import org.apache.hadoop.hbase.util.Bytes;
038import org.apache.hadoop.hbase.util.FSTableDescriptors;
039import org.apache.hadoop.hbase.util.FSUtils;
040import org.apache.hadoop.hdfs.MiniDFSCluster;
041import org.slf4j.Logger;
042import org.slf4j.LoggerFactory;
043
044import junit.framework.AssertionFailedError;
045import junit.framework.TestCase;
046
047/**
048 * Abstract HBase test class.  Initializes a few things that can come in handly
049 * like an HBaseConfiguration and filesystem.
050 * @deprecated since 2.0.0 and will be removed in 3.0.0. Write junit4 unit tests using
051 *   {@link HBaseTestingUtility}.
052 * @see HBaseTestingUtility
053 * @see <a href="https://issues.apache.org/jira/browse/HBASE-11912">HBASE-11912</a>
054 */
055@Deprecated
056public abstract class HBaseTestCase extends TestCase {
057  private static final Logger LOG = LoggerFactory.getLogger(HBaseTestCase.class);
058
059  protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
060  protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
061  protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
062
063  protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
064
065  private boolean localfs = false;
066  protected static Path testDir = null;
067  protected FileSystem fs = null;
068  protected HRegion meta = null;
069  protected static final char FIRST_CHAR = 'a';
070  protected static final char LAST_CHAR = 'z';
071  protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
072  protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
073  protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
074  protected static final int MAXVERSIONS = 3;
075
076  protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
077
078  public volatile Configuration conf = testUtil.getConfiguration();
079  public final FSTableDescriptors fsTableDescriptors;
080  {
081    try {
082      fsTableDescriptors = new FSTableDescriptors(conf);
083    } catch (IOException e) {
084      throw new RuntimeException("Failed to init descriptors", e);
085    }
086  }
087
088  /** constructor */
089  public HBaseTestCase() {
090    super();
091  }
092
093  /**
094   * @param name
095   */
096  public HBaseTestCase(String name) {
097    super(name);
098  }
099
100  /**
101   * Note that this method must be called after the mini hdfs cluster has
102   * started or we end up with a local file system.
103   */
104  @Override
105  protected void setUp() throws Exception {
106    super.setUp();
107    localfs =
108      (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
109
110    if (fs == null) {
111      this.fs = FileSystem.get(conf);
112    }
113    try {
114      if (localfs) {
115        testDir = getUnitTestdir(getName());
116        if (fs.exists(testDir)) {
117          fs.delete(testDir, true);
118        }
119      } else {
120        testDir = FSUtils.getRootDir(conf);
121      }
122    } catch (Exception e) {
123      LOG.error(HBaseMarkers.FATAL, "error during setup", e);
124      throw e;
125    }
126  }
127
128  @Override
129  protected void tearDown() throws Exception {
130    try {
131      if (localfs) {
132        if (this.fs.exists(testDir)) {
133          this.fs.delete(testDir, true);
134        }
135      }
136    } catch (Exception e) {
137      LOG.error(HBaseMarkers.FATAL, "error during tear down", e);
138    }
139    super.tearDown();
140  }
141
142  /**
143   * @see HBaseTestingUtility#getBaseTestDir
144   * @param testName
145   * @return directory to use for this test
146   */
147    protected Path getUnitTestdir(String testName) {
148      return testUtil.getDataTestDir(testName);
149    }
150
151  /**
152   * You must call close on the returned region and then close on the log file it created. Do
153   * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} to close both the region and the WAL.
154   * @param desc
155   * @param startKey
156   * @param endKey
157   * @return An {@link HRegion}
158   * @throws IOException
159   */
160  public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
161      byte [] endKey)
162  throws IOException {
163    return createNewHRegion(desc, startKey, endKey, this.conf);
164  }
165
166  public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
167      byte [] endKey, Configuration conf)
168  throws IOException {
169    HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
170    return HBaseTestingUtility.createRegionAndWAL(hri, testDir, conf, desc);
171  }
172
173  protected HRegion openClosedRegion(final HRegion closedRegion)
174  throws IOException {
175    return HRegion.openHRegion(closedRegion, null);
176  }
177
178  /**
179   * Create a table of name <code>name</code> with {@link COLUMNS} for
180   * families.
181   * @param name Name to give table.
182   * @return Column descriptor.
183   */
184  protected HTableDescriptor createTableDescriptor(final String name) {
185    return createTableDescriptor(name, MAXVERSIONS);
186  }
187
188  /**
189   * Create a table of name <code>name</code> with {@link COLUMNS} for
190   * families.
191   * @param name Name to give table.
192   * @param versions How many versions to allow per column.
193   * @return Column descriptor.
194   */
195  protected HTableDescriptor createTableDescriptor(final String name,
196      final int versions) {
197    return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
198        versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
199  }
200
201  /**
202   * Create a table of name <code>name</code> with {@link COLUMNS} for
203   * families.
204   * @param name Name to give table.
205   * @param versions How many versions to allow per column.
206   * @return Column descriptor.
207   */
208  protected HTableDescriptor createTableDescriptor(final String name,
209      final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) {
210    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
211    for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
212      htd.addFamily(new HColumnDescriptor(cfName)
213          .setMinVersions(minVersions)
214          .setMaxVersions(versions)
215          .setKeepDeletedCells(keepDeleted)
216          .setBlockCacheEnabled(false)
217          .setTimeToLive(ttl)
218      );
219    }
220    return htd;
221  }
222
223  /**
224   * Add content to region <code>r</code> on the passed column
225   * <code>column</code>.
226   * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
227   * @param r
228   * @param columnFamily
229   * @param column
230   * @throws IOException
231   * @return count of what we added.
232   */
233  public static long addContent(final Region r, final byte [] columnFamily, final byte[] column)
234  throws IOException {
235    byte [] startKey = r.getRegionInfo().getStartKey();
236    byte [] endKey = r.getRegionInfo().getEndKey();
237    byte [] startKeyBytes = startKey;
238    if (startKeyBytes == null || startKeyBytes.length == 0) {
239      startKeyBytes = START_KEY_BYTES;
240    }
241    return addContent(new RegionAsTable(r), Bytes.toString(columnFamily), Bytes.toString(column),
242      startKeyBytes, endKey, -1);
243  }
244
245  public static long addContent(final Region r, final byte [] columnFamily) throws IOException {
246    return addContent(r, columnFamily, null);
247  }
248
249  /**
250   * Add content to region <code>r</code> on the passed column
251   * <code>column</code>.
252   * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
253   * @throws IOException
254   * @return count of what we added.
255   */
256  public static long addContent(final Table updater,
257      final String columnFamily) throws IOException {
258    return addContent(updater, columnFamily, START_KEY_BYTES, null);
259  }
260
261  public static long addContent(final Table updater, final String family,
262      final String column) throws IOException {
263    return addContent(updater, family, column, START_KEY_BYTES, null);
264  }
265
266  /**
267   * Add content to region <code>r</code> on the passed column
268   * <code>column</code>.
269   * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
270   * @return count of what we added.
271   * @throws IOException
272   */
273  public static long addContent(final Table updater, final String columnFamily,
274      final byte [] startKeyBytes, final byte [] endKey)
275  throws IOException {
276    return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
277  }
278
279  public static long addContent(final Table updater, final String family, String column,
280      final byte [] startKeyBytes, final byte [] endKey) throws IOException {
281    return addContent(updater, family, column, startKeyBytes, endKey, -1);
282  }
283
284  /**
285   * Add content to region <code>r</code> on the passed column
286   * <code>column</code>.
287   * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
288   * @return count of what we added.
289   * @throws IOException
290   */
291  public static long addContent(final Table updater,
292                                   final String columnFamily,
293                                   final String column,
294      final byte [] startKeyBytes, final byte [] endKey, final long ts)
295  throws IOException {
296    long count = 0;
297    // Add rows of three characters.  The first character starts with the
298    // 'a' character and runs up to 'z'.  Per first character, we run the
299    // second character over same range.  And same for the third so rows
300    // (and values) look like this: 'aaa', 'aab', 'aac', etc.
301    char secondCharStart = (char)startKeyBytes[1];
302    char thirdCharStart = (char)startKeyBytes[2];
303    EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
304      for (char d = secondCharStart; d <= LAST_CHAR; d++) {
305        for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
306          byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
307          if (endKey != null && endKey.length > 0
308              && Bytes.compareTo(endKey, t) <= 0) {
309            break EXIT;
310          }
311          try {
312            Put put;
313            if(ts != -1) {
314              put = new Put(t, ts);
315            } else {
316              put = new Put(t);
317            }
318            try {
319              StringBuilder sb = new StringBuilder();
320              if (column != null && column.contains(":")) {
321                sb.append(column);
322              } else {
323                if (columnFamily != null) {
324                  sb.append(columnFamily);
325                  if (!columnFamily.endsWith(":")) {
326                    sb.append(":");
327                  }
328                  if (column != null) {
329                    sb.append(column);
330                  }
331                }
332              }
333              byte[][] split =
334                CellUtil.parseColumn(Bytes.toBytes(sb.toString()));
335              if(split.length == 1) {
336                byte[] qualifier = new byte[0];
337                put.addColumn(split[0], qualifier, t);
338              } else {
339                put.addColumn(split[0], split[1], t);
340              }
341              put.setDurability(Durability.SKIP_WAL);
342              updater.put(put);
343              count++;
344            } catch (RuntimeException ex) {
345              ex.printStackTrace();
346              throw ex;
347            } catch (IOException ex) {
348              ex.printStackTrace();
349              throw ex;
350            }
351          } catch (RuntimeException ex) {
352            ex.printStackTrace();
353            throw ex;
354          } catch (IOException ex) {
355            ex.printStackTrace();
356            throw ex;
357          }
358        }
359        // Set start character back to FIRST_CHAR after we've done first loop.
360        thirdCharStart = FIRST_CHAR;
361      }
362      secondCharStart = FIRST_CHAR;
363    }
364    return count;
365  }
366
367  protected void assertResultEquals(final HRegion region, final byte [] row,
368      final byte [] family, final byte [] qualifier, final long timestamp,
369      final byte [] value) throws IOException {
370    Get get = new Get(row);
371    get.setTimestamp(timestamp);
372    Result res = region.get(get);
373    NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
374      res.getMap();
375    byte [] res_value = map.get(family).get(qualifier).get(timestamp);
376
377    if (value == null) {
378      assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
379          " at timestamp " + timestamp, null, res_value);
380    } else {
381      if (res_value == null) {
382        fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
383            " at timestamp " + timestamp + "\" was expected to be \"" +
384            Bytes.toStringBinary(value) + " but was null");
385      }
386      if (res_value != null) {
387        assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
388            " at timestamp " +
389            timestamp, value, new String(res_value, StandardCharsets.UTF_8));
390      }
391    }
392  }
393
394  /**
395   * Common method to close down a MiniDFSCluster and the associated file system
396   *
397   * @param cluster
398   */
399  public static void shutdownDfs(MiniDFSCluster cluster) {
400    if (cluster != null) {
401      LOG.info("Shutting down Mini DFS ");
402      try {
403        cluster.shutdown();
404      } catch (Exception e) {
405        /// Can get a java.lang.reflect.UndeclaredThrowableException thrown
406        // here because of an InterruptedException. Don't let exceptions in
407        // here be cause of test failure.
408      }
409      try {
410        FileSystem fs = cluster.getFileSystem();
411        if (fs != null) {
412          LOG.info("Shutting down FileSystem");
413          fs.close();
414        }
415        FileSystem.closeAll();
416      } catch (IOException e) {
417        LOG.error("error closing file system", e);
418      }
419    }
420  }
421
422  /**
423   * You must call {@link #closeRootAndMeta()} when done after calling this
424   * method. It does cleanup.
425   * @throws IOException
426   */
427  protected void createMetaRegion() throws IOException {
428    FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(conf);
429    meta = HBaseTestingUtility.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO, testDir,
430        conf, fsTableDescriptors.get(TableName.META_TABLE_NAME));
431  }
432
433  protected void closeRootAndMeta() throws IOException {
434    HBaseTestingUtility.closeRegionAndWAL(meta);
435  }
436
437  public static void assertByteEquals(byte[] expected,
438                               byte[] actual) {
439    if (Bytes.compareTo(expected, actual) != 0) {
440      throw new AssertionFailedError("expected:<" +
441      Bytes.toString(expected) + "> but was:<" +
442      Bytes.toString(actual) + ">");
443    }
444  }
445
446  public static void assertEquals(byte[] expected,
447                               byte[] actual) {
448    if (Bytes.compareTo(expected, actual) != 0) {
449      throw new AssertionFailedError("expected:<" +
450      Bytes.toStringBinary(expected) + "> but was:<" +
451      Bytes.toStringBinary(actual) + ">");
452    }
453  }
454}