001/**
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019package org.apache.hadoop.hbase;
020
021import java.io.IOException;
022import java.nio.charset.StandardCharsets;
023import java.util.NavigableMap;
024import junit.framework.AssertionFailedError;
025import junit.framework.TestCase;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.fs.FileSystem;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.client.Durability;
030import org.apache.hadoop.hbase.client.Get;
031import org.apache.hadoop.hbase.client.Put;
032import org.apache.hadoop.hbase.client.Result;
033import org.apache.hadoop.hbase.client.Table;
034import org.apache.hadoop.hbase.log.HBaseMarkers;
035import org.apache.hadoop.hbase.regionserver.HRegion;
036import org.apache.hadoop.hbase.regionserver.Region;
037import org.apache.hadoop.hbase.regionserver.RegionAsTable;
038import org.apache.hadoop.hbase.util.Bytes;
039import org.apache.hadoop.hbase.util.CommonFSUtils;
040import org.apache.hadoop.hbase.util.FSTableDescriptors;
041import org.apache.hadoop.hdfs.MiniDFSCluster;
042import org.slf4j.Logger;
043import org.slf4j.LoggerFactory;
044
045/**
046 * Abstract HBase test class.  Initializes a few things that can come in handly
047 * like an HBaseConfiguration and filesystem.
048 * @deprecated since 2.0.0 and will be removed in 3.0.0. Write junit4 unit tests using
049 *   {@link HBaseTestingUtility}.
050 * @see HBaseTestingUtility
051 * @see <a href="https://issues.apache.org/jira/browse/HBASE-11912">HBASE-11912</a>
052 */
053@Deprecated
054public abstract class HBaseTestCase extends TestCase {
055  private static final Logger LOG = LoggerFactory.getLogger(HBaseTestCase.class);
056
057  protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
058  protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
059  protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
060
061  protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
062
063  private boolean localfs = false;
064  protected static Path testDir = null;
065  protected FileSystem fs = null;
066  protected HRegion meta = null;
067  protected static final char FIRST_CHAR = 'a';
068  protected static final char LAST_CHAR = 'z';
069  protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
070  protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
071  protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
072  protected static final int MAXVERSIONS = 3;
073
074  protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
075
076  public volatile Configuration conf = testUtil.getConfiguration();
077  public final FSTableDescriptors fsTableDescriptors;
078  {
079    try {
080      fsTableDescriptors = new FSTableDescriptors(conf);
081    } catch (IOException e) {
082      throw new RuntimeException("Failed to init descriptors", e);
083    }
084  }
085
086  /** constructor */
087  public HBaseTestCase() {
088    super();
089  }
090
091  /**
092   * @param name
093   */
094  public HBaseTestCase(String name) {
095    super(name);
096  }
097
098  /**
099   * Note that this method must be called after the mini hdfs cluster has
100   * started or we end up with a local file system.
101   */
102  @Override
103  protected void setUp() throws Exception {
104    super.setUp();
105    localfs =
106      (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
107
108    if (fs == null) {
109      this.fs = FileSystem.get(conf);
110    }
111    try {
112      if (localfs) {
113        testDir = getUnitTestdir(getName());
114        if (fs.exists(testDir)) {
115          fs.delete(testDir, true);
116        }
117      } else {
118        testDir = CommonFSUtils.getRootDir(conf);
119      }
120    } catch (Exception e) {
121      LOG.error(HBaseMarkers.FATAL, "error during setup", e);
122      throw e;
123    }
124  }
125
126  @Override
127  protected void tearDown() throws Exception {
128    try {
129      if (localfs) {
130        if (this.fs.exists(testDir)) {
131          this.fs.delete(testDir, true);
132        }
133      }
134    } catch (Exception e) {
135      LOG.error(HBaseMarkers.FATAL, "error during tear down", e);
136    }
137    super.tearDown();
138  }
139
140  /**
141   * @see HBaseTestingUtility#getBaseTestDir
142   * @param testName
143   * @return directory to use for this test
144   */
145    protected Path getUnitTestdir(String testName) {
146      return testUtil.getDataTestDir(testName);
147    }
148
149  /**
150   * You must call close on the returned region and then close on the log file it created. Do
151   * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} to close both the region and the WAL.
152   * @param desc
153   * @param startKey
154   * @param endKey
155   * @return An {@link HRegion}
156   * @throws IOException
157   */
158  public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
159      byte [] endKey)
160  throws IOException {
161    return createNewHRegion(desc, startKey, endKey, this.conf);
162  }
163
164  public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
165      byte [] endKey, Configuration conf)
166  throws IOException {
167    HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
168    return HBaseTestingUtility.createRegionAndWAL(hri, testDir, conf, desc);
169  }
170
171  protected HRegion openClosedRegion(final HRegion closedRegion)
172  throws IOException {
173    return HRegion.openHRegion(closedRegion, null);
174  }
175
176  /**
177   * Create a table of name {@code name} with {@link #COLUMNS} for
178   * families.
179   * @param name Name to give table.
180   * @return Column descriptor.
181   */
182  protected HTableDescriptor createTableDescriptor(final String name) {
183    return createTableDescriptor(name, MAXVERSIONS);
184  }
185
186  /**
187   * Create a table of name {@code name} with {@link #COLUMNS} for
188   * families.
189   * @param name Name to give table.
190   * @param versions How many versions to allow per column.
191   * @return Column descriptor.
192   */
193  protected HTableDescriptor createTableDescriptor(final String name,
194      final int versions) {
195    return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
196        versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
197  }
198
199  /**
200   * Create a table of name {@code name} with {@link #COLUMNS} for
201   * families.
202   * @param name Name to give table.
203   * @param versions How many versions to allow per column.
204   * @return Column descriptor.
205   */
206  protected HTableDescriptor createTableDescriptor(final String name,
207      final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) {
208    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
209    for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
210      htd.addFamily(new HColumnDescriptor(cfName)
211          .setMinVersions(minVersions)
212          .setMaxVersions(versions)
213          .setKeepDeletedCells(keepDeleted)
214          .setBlockCacheEnabled(false)
215          .setTimeToLive(ttl)
216      );
217    }
218    return htd;
219  }
220
221  /**
222   * Add content to region <code>r</code> on the passed column
223   * <code>column</code>.
224   * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
225   * @param r
226   * @param columnFamily
227   * @param column
228   * @throws IOException
229   * @return count of what we added.
230   */
231  public static long addContent(final Region r, final byte [] columnFamily, final byte[] column)
232  throws IOException {
233    byte [] startKey = r.getRegionInfo().getStartKey();
234    byte [] endKey = r.getRegionInfo().getEndKey();
235    byte [] startKeyBytes = startKey;
236    if (startKeyBytes == null || startKeyBytes.length == 0) {
237      startKeyBytes = START_KEY_BYTES;
238    }
239    return addContent(new RegionAsTable(r), Bytes.toString(columnFamily), Bytes.toString(column),
240      startKeyBytes, endKey, -1);
241  }
242
243  public static long addContent(final Region r, final byte [] columnFamily) throws IOException {
244    return addContent(r, columnFamily, null);
245  }
246
247  /**
248   * Add content to region <code>r</code> on the passed column
249   * <code>column</code>.
250   * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
251   * @throws IOException
252   * @return count of what we added.
253   */
254  public static long addContent(final Table updater,
255      final String columnFamily) throws IOException {
256    return addContent(updater, columnFamily, START_KEY_BYTES, null);
257  }
258
259  public static long addContent(final Table updater, final String family,
260      final String column) throws IOException {
261    return addContent(updater, family, column, START_KEY_BYTES, null);
262  }
263
264  /**
265   * Add content to region <code>r</code> on the passed column
266   * <code>column</code>.
267   * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
268   * @return count of what we added.
269   * @throws IOException
270   */
271  public static long addContent(final Table updater, final String columnFamily,
272      final byte [] startKeyBytes, final byte [] endKey)
273  throws IOException {
274    return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
275  }
276
277  public static long addContent(final Table updater, final String family, String column,
278      final byte [] startKeyBytes, final byte [] endKey) throws IOException {
279    return addContent(updater, family, column, startKeyBytes, endKey, -1);
280  }
281
282  /**
283   * Add content to region <code>r</code> on the passed column
284   * <code>column</code>.
285   * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
286   * @return count of what we added.
287   * @throws IOException
288   */
289  public static long addContent(final Table updater,
290                                   final String columnFamily,
291                                   final String column,
292      final byte [] startKeyBytes, final byte [] endKey, final long ts)
293  throws IOException {
294    long count = 0;
295    // Add rows of three characters.  The first character starts with the
296    // 'a' character and runs up to 'z'.  Per first character, we run the
297    // second character over same range.  And same for the third so rows
298    // (and values) look like this: 'aaa', 'aab', 'aac', etc.
299    char secondCharStart = (char)startKeyBytes[1];
300    char thirdCharStart = (char)startKeyBytes[2];
301    EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
302      for (char d = secondCharStart; d <= LAST_CHAR; d++) {
303        for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
304          byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
305          if (endKey != null && endKey.length > 0
306              && Bytes.compareTo(endKey, t) <= 0) {
307            break EXIT;
308          }
309          try {
310            Put put;
311            if(ts != -1) {
312              put = new Put(t, ts);
313            } else {
314              put = new Put(t);
315            }
316            try {
317              StringBuilder sb = new StringBuilder();
318              if (column != null && column.contains(":")) {
319                sb.append(column);
320              } else {
321                if (columnFamily != null) {
322                  sb.append(columnFamily);
323                  if (!columnFamily.endsWith(":")) {
324                    sb.append(":");
325                  }
326                  if (column != null) {
327                    sb.append(column);
328                  }
329                }
330              }
331              byte[][] split =
332                CellUtil.parseColumn(Bytes.toBytes(sb.toString()));
333              if(split.length == 1) {
334                byte[] qualifier = new byte[0];
335                put.addColumn(split[0], qualifier, t);
336              } else {
337                put.addColumn(split[0], split[1], t);
338              }
339              put.setDurability(Durability.SKIP_WAL);
340              updater.put(put);
341              count++;
342            } catch (RuntimeException ex) {
343              ex.printStackTrace();
344              throw ex;
345            } catch (IOException ex) {
346              ex.printStackTrace();
347              throw ex;
348            }
349          } catch (RuntimeException ex) {
350            ex.printStackTrace();
351            throw ex;
352          } catch (IOException ex) {
353            ex.printStackTrace();
354            throw ex;
355          }
356        }
357        // Set start character back to FIRST_CHAR after we've done first loop.
358        thirdCharStart = FIRST_CHAR;
359      }
360      secondCharStart = FIRST_CHAR;
361    }
362    return count;
363  }
364
365  protected void assertResultEquals(final HRegion region, final byte [] row,
366      final byte [] family, final byte [] qualifier, final long timestamp,
367      final byte [] value) throws IOException {
368    Get get = new Get(row);
369    get.setTimestamp(timestamp);
370    Result res = region.get(get);
371    NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
372      res.getMap();
373    byte [] res_value = map.get(family).get(qualifier).get(timestamp);
374
375    if (value == null) {
376      assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
377          " at timestamp " + timestamp, null, res_value);
378    } else {
379      if (res_value == null) {
380        fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
381            " at timestamp " + timestamp + "\" was expected to be \"" +
382            Bytes.toStringBinary(value) + " but was null");
383      }
384      if (res_value != null) {
385        assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
386            " at timestamp " +
387            timestamp, value, new String(res_value, StandardCharsets.UTF_8));
388      }
389    }
390  }
391
392  /**
393   * Common method to close down a MiniDFSCluster and the associated file system
394   *
395   * @param cluster
396   */
397  public static void shutdownDfs(MiniDFSCluster cluster) {
398    if (cluster != null) {
399      LOG.info("Shutting down Mini DFS ");
400      try {
401        cluster.shutdown();
402      } catch (Exception e) {
403        /// Can get a java.lang.reflect.UndeclaredThrowableException thrown
404        // here because of an InterruptedException. Don't let exceptions in
405        // here be cause of test failure.
406      }
407      try {
408        FileSystem fs = cluster.getFileSystem();
409        if (fs != null) {
410          LOG.info("Shutting down FileSystem");
411          fs.close();
412        }
413        FileSystem.closeAll();
414      } catch (IOException e) {
415        LOG.error("error closing file system", e);
416      }
417    }
418  }
419
420  /**
421   * You must call {@link #closeRootAndMeta()} when done after calling this
422   * method. It does cleanup.
423   * @throws IOException
424   */
425  protected void createMetaRegion() throws IOException {
426    FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(conf);
427    meta = HBaseTestingUtility.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO, testDir,
428        conf, fsTableDescriptors.get(TableName.META_TABLE_NAME));
429  }
430
431  protected void closeRootAndMeta() throws IOException {
432    HBaseTestingUtility.closeRegionAndWAL(meta);
433  }
434
435  public static void assertByteEquals(byte[] expected,
436                               byte[] actual) {
437    if (Bytes.compareTo(expected, actual) != 0) {
438      throw new AssertionFailedError("expected:<" +
439      Bytes.toString(expected) + "> but was:<" +
440      Bytes.toString(actual) + ">");
441    }
442  }
443
444  public static void assertEquals(byte[] expected,
445                               byte[] actual) {
446    if (Bytes.compareTo(expected, actual) != 0) {
447      throw new AssertionFailedError("expected:<" +
448      Bytes.toStringBinary(expected) + "> but was:<" +
449      Bytes.toStringBinary(actual) + ">");
450    }
451  }
452}