001/** 002 * 003 * Licensed to the Apache Software Foundation (ASF) under one 004 * or more contributor license agreements. See the NOTICE file 005 * distributed with this work for additional information 006 * regarding copyright ownership. The ASF licenses this file 007 * to you under the Apache License, Version 2.0 (the 008 * "License"); you may not use this file except in compliance 009 * with the License. You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 */ 019package org.apache.hadoop.hbase; 020 021import java.io.IOException; 022import java.nio.charset.StandardCharsets; 023import java.util.NavigableMap; 024import junit.framework.AssertionFailedError; 025import junit.framework.TestCase; 026import org.apache.hadoop.conf.Configuration; 027import org.apache.hadoop.fs.FileSystem; 028import org.apache.hadoop.fs.Path; 029import org.apache.hadoop.hbase.client.Durability; 030import org.apache.hadoop.hbase.client.Get; 031import org.apache.hadoop.hbase.client.Put; 032import org.apache.hadoop.hbase.client.Result; 033import org.apache.hadoop.hbase.client.Table; 034import org.apache.hadoop.hbase.log.HBaseMarkers; 035import org.apache.hadoop.hbase.regionserver.HRegion; 036import org.apache.hadoop.hbase.regionserver.Region; 037import org.apache.hadoop.hbase.regionserver.RegionAsTable; 038import org.apache.hadoop.hbase.util.Bytes; 039import org.apache.hadoop.hbase.util.CommonFSUtils; 040import org.apache.hadoop.hbase.util.FSTableDescriptors; 041import org.apache.hadoop.hdfs.MiniDFSCluster; 042import org.slf4j.Logger; 043import org.slf4j.LoggerFactory; 044 045/** 046 * Abstract HBase test class. Initializes a few things that can come in handly 047 * like an HBaseConfiguration and filesystem. 048 * @deprecated since 2.0.0 and will be removed in 3.0.0. Write junit4 unit tests using 049 * {@link HBaseTestingUtility}. 050 * @see HBaseTestingUtility 051 * @see <a href="https://issues.apache.org/jira/browse/HBASE-11912">HBASE-11912</a> 052 */ 053@Deprecated 054public abstract class HBaseTestCase extends TestCase { 055 private static final Logger LOG = LoggerFactory.getLogger(HBaseTestCase.class); 056 057 protected final static byte [] fam1 = Bytes.toBytes("colfamily11"); 058 protected final static byte [] fam2 = Bytes.toBytes("colfamily21"); 059 protected final static byte [] fam3 = Bytes.toBytes("colfamily31"); 060 061 protected static final byte [][] COLUMNS = {fam1, fam2, fam3}; 062 063 private boolean localfs = false; 064 protected static Path testDir = null; 065 protected FileSystem fs = null; 066 protected HRegion meta = null; 067 protected static final char FIRST_CHAR = 'a'; 068 protected static final char LAST_CHAR = 'z'; 069 protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|"; 070 protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR}; 071 protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET); 072 protected static final int MAXVERSIONS = 3; 073 074 protected final HBaseTestingUtility testUtil = new HBaseTestingUtility(); 075 076 public volatile Configuration conf = testUtil.getConfiguration(); 077 public final FSTableDescriptors fsTableDescriptors; 078 { 079 try { 080 fsTableDescriptors = new FSTableDescriptors(conf); 081 } catch (IOException e) { 082 throw new RuntimeException("Failed to init descriptors", e); 083 } 084 } 085 086 /** constructor */ 087 public HBaseTestCase() { 088 super(); 089 } 090 091 /** 092 * @param name 093 */ 094 public HBaseTestCase(String name) { 095 super(name); 096 } 097 098 /** 099 * Note that this method must be called after the mini hdfs cluster has 100 * started or we end up with a local file system. 101 */ 102 @Override 103 protected void setUp() throws Exception { 104 super.setUp(); 105 localfs = 106 (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0); 107 108 if (fs == null) { 109 this.fs = FileSystem.get(conf); 110 } 111 try { 112 if (localfs) { 113 testDir = getUnitTestdir(getName()); 114 if (fs.exists(testDir)) { 115 fs.delete(testDir, true); 116 } 117 } else { 118 testDir = CommonFSUtils.getRootDir(conf); 119 } 120 } catch (Exception e) { 121 LOG.error(HBaseMarkers.FATAL, "error during setup", e); 122 throw e; 123 } 124 } 125 126 @Override 127 protected void tearDown() throws Exception { 128 try { 129 if (localfs) { 130 if (this.fs.exists(testDir)) { 131 this.fs.delete(testDir, true); 132 } 133 } 134 } catch (Exception e) { 135 LOG.error(HBaseMarkers.FATAL, "error during tear down", e); 136 } 137 super.tearDown(); 138 } 139 140 /** 141 * @see HBaseTestingUtility#getBaseTestDir 142 * @param testName 143 * @return directory to use for this test 144 */ 145 protected Path getUnitTestdir(String testName) { 146 return testUtil.getDataTestDir(testName); 147 } 148 149 /** 150 * You must call close on the returned region and then close on the log file it created. Do 151 * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} to close both the region and the WAL. 152 * @param desc 153 * @param startKey 154 * @param endKey 155 * @return An {@link HRegion} 156 * @throws IOException 157 */ 158 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey, 159 byte [] endKey) 160 throws IOException { 161 return createNewHRegion(desc, startKey, endKey, this.conf); 162 } 163 164 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey, 165 byte [] endKey, Configuration conf) 166 throws IOException { 167 HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey); 168 return HBaseTestingUtility.createRegionAndWAL(hri, testDir, conf, desc); 169 } 170 171 protected HRegion openClosedRegion(final HRegion closedRegion) 172 throws IOException { 173 return HRegion.openHRegion(closedRegion, null); 174 } 175 176 /** 177 * Create a table of name {@code name} with {@link #COLUMNS} for 178 * families. 179 * @param name Name to give table. 180 * @return Column descriptor. 181 */ 182 protected HTableDescriptor createTableDescriptor(final String name) { 183 return createTableDescriptor(name, MAXVERSIONS); 184 } 185 186 /** 187 * Create a table of name {@code name} with {@link #COLUMNS} for 188 * families. 189 * @param name Name to give table. 190 * @param versions How many versions to allow per column. 191 * @return Column descriptor. 192 */ 193 protected HTableDescriptor createTableDescriptor(final String name, 194 final int versions) { 195 return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS, 196 versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED); 197 } 198 199 /** 200 * Create a table of name {@code name} with {@link #COLUMNS} for 201 * families. 202 * @param name Name to give table. 203 * @param versions How many versions to allow per column. 204 * @return Column descriptor. 205 */ 206 protected HTableDescriptor createTableDescriptor(final String name, 207 final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) { 208 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); 209 for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) { 210 htd.addFamily(new HColumnDescriptor(cfName) 211 .setMinVersions(minVersions) 212 .setMaxVersions(versions) 213 .setKeepDeletedCells(keepDeleted) 214 .setBlockCacheEnabled(false) 215 .setTimeToLive(ttl) 216 ); 217 } 218 return htd; 219 } 220 221 /** 222 * Add content to region <code>r</code> on the passed column 223 * <code>column</code>. 224 * Adds data of the from 'aaa', 'aab', etc where key and value are the same. 225 * @param r 226 * @param columnFamily 227 * @param column 228 * @throws IOException 229 * @return count of what we added. 230 */ 231 public static long addContent(final Region r, final byte [] columnFamily, final byte[] column) 232 throws IOException { 233 byte [] startKey = r.getRegionInfo().getStartKey(); 234 byte [] endKey = r.getRegionInfo().getEndKey(); 235 byte [] startKeyBytes = startKey; 236 if (startKeyBytes == null || startKeyBytes.length == 0) { 237 startKeyBytes = START_KEY_BYTES; 238 } 239 return addContent(new RegionAsTable(r), Bytes.toString(columnFamily), Bytes.toString(column), 240 startKeyBytes, endKey, -1); 241 } 242 243 public static long addContent(final Region r, final byte [] columnFamily) throws IOException { 244 return addContent(r, columnFamily, null); 245 } 246 247 /** 248 * Add content to region <code>r</code> on the passed column 249 * <code>column</code>. 250 * Adds data of the from 'aaa', 'aab', etc where key and value are the same. 251 * @throws IOException 252 * @return count of what we added. 253 */ 254 public static long addContent(final Table updater, 255 final String columnFamily) throws IOException { 256 return addContent(updater, columnFamily, START_KEY_BYTES, null); 257 } 258 259 public static long addContent(final Table updater, final String family, 260 final String column) throws IOException { 261 return addContent(updater, family, column, START_KEY_BYTES, null); 262 } 263 264 /** 265 * Add content to region <code>r</code> on the passed column 266 * <code>column</code>. 267 * Adds data of the from 'aaa', 'aab', etc where key and value are the same. 268 * @return count of what we added. 269 * @throws IOException 270 */ 271 public static long addContent(final Table updater, final String columnFamily, 272 final byte [] startKeyBytes, final byte [] endKey) 273 throws IOException { 274 return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1); 275 } 276 277 public static long addContent(final Table updater, final String family, String column, 278 final byte [] startKeyBytes, final byte [] endKey) throws IOException { 279 return addContent(updater, family, column, startKeyBytes, endKey, -1); 280 } 281 282 /** 283 * Add content to region <code>r</code> on the passed column 284 * <code>column</code>. 285 * Adds data of the from 'aaa', 'aab', etc where key and value are the same. 286 * @return count of what we added. 287 * @throws IOException 288 */ 289 public static long addContent(final Table updater, 290 final String columnFamily, 291 final String column, 292 final byte [] startKeyBytes, final byte [] endKey, final long ts) 293 throws IOException { 294 long count = 0; 295 // Add rows of three characters. The first character starts with the 296 // 'a' character and runs up to 'z'. Per first character, we run the 297 // second character over same range. And same for the third so rows 298 // (and values) look like this: 'aaa', 'aab', 'aac', etc. 299 char secondCharStart = (char)startKeyBytes[1]; 300 char thirdCharStart = (char)startKeyBytes[2]; 301 EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) { 302 for (char d = secondCharStart; d <= LAST_CHAR; d++) { 303 for (char e = thirdCharStart; e <= LAST_CHAR; e++) { 304 byte [] t = new byte [] {(byte)c, (byte)d, (byte)e}; 305 if (endKey != null && endKey.length > 0 306 && Bytes.compareTo(endKey, t) <= 0) { 307 break EXIT; 308 } 309 try { 310 Put put; 311 if(ts != -1) { 312 put = new Put(t, ts); 313 } else { 314 put = new Put(t); 315 } 316 try { 317 StringBuilder sb = new StringBuilder(); 318 if (column != null && column.contains(":")) { 319 sb.append(column); 320 } else { 321 if (columnFamily != null) { 322 sb.append(columnFamily); 323 if (!columnFamily.endsWith(":")) { 324 sb.append(":"); 325 } 326 if (column != null) { 327 sb.append(column); 328 } 329 } 330 } 331 byte[][] split = 332 CellUtil.parseColumn(Bytes.toBytes(sb.toString())); 333 if(split.length == 1) { 334 byte[] qualifier = new byte[0]; 335 put.addColumn(split[0], qualifier, t); 336 } else { 337 put.addColumn(split[0], split[1], t); 338 } 339 put.setDurability(Durability.SKIP_WAL); 340 updater.put(put); 341 count++; 342 } catch (RuntimeException ex) { 343 ex.printStackTrace(); 344 throw ex; 345 } catch (IOException ex) { 346 ex.printStackTrace(); 347 throw ex; 348 } 349 } catch (RuntimeException ex) { 350 ex.printStackTrace(); 351 throw ex; 352 } catch (IOException ex) { 353 ex.printStackTrace(); 354 throw ex; 355 } 356 } 357 // Set start character back to FIRST_CHAR after we've done first loop. 358 thirdCharStart = FIRST_CHAR; 359 } 360 secondCharStart = FIRST_CHAR; 361 } 362 return count; 363 } 364 365 protected void assertResultEquals(final HRegion region, final byte [] row, 366 final byte [] family, final byte [] qualifier, final long timestamp, 367 final byte [] value) throws IOException { 368 Get get = new Get(row); 369 get.setTimestamp(timestamp); 370 Result res = region.get(get); 371 NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map = 372 res.getMap(); 373 byte [] res_value = map.get(family).get(qualifier).get(timestamp); 374 375 if (value == null) { 376 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) + 377 " at timestamp " + timestamp, null, res_value); 378 } else { 379 if (res_value == null) { 380 fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) + 381 " at timestamp " + timestamp + "\" was expected to be \"" + 382 Bytes.toStringBinary(value) + " but was null"); 383 } 384 if (res_value != null) { 385 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) + 386 " at timestamp " + 387 timestamp, value, new String(res_value, StandardCharsets.UTF_8)); 388 } 389 } 390 } 391 392 /** 393 * Common method to close down a MiniDFSCluster and the associated file system 394 * 395 * @param cluster 396 */ 397 public static void shutdownDfs(MiniDFSCluster cluster) { 398 if (cluster != null) { 399 LOG.info("Shutting down Mini DFS "); 400 try { 401 cluster.shutdown(); 402 } catch (Exception e) { 403 /// Can get a java.lang.reflect.UndeclaredThrowableException thrown 404 // here because of an InterruptedException. Don't let exceptions in 405 // here be cause of test failure. 406 } 407 try { 408 FileSystem fs = cluster.getFileSystem(); 409 if (fs != null) { 410 LOG.info("Shutting down FileSystem"); 411 fs.close(); 412 } 413 FileSystem.closeAll(); 414 } catch (IOException e) { 415 LOG.error("error closing file system", e); 416 } 417 } 418 } 419 420 /** 421 * You must call {@link #closeRootAndMeta()} when done after calling this 422 * method. It does cleanup. 423 * @throws IOException 424 */ 425 protected void createMetaRegion() throws IOException { 426 FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(conf); 427 meta = HBaseTestingUtility.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO, testDir, 428 conf, fsTableDescriptors.get(TableName.META_TABLE_NAME)); 429 } 430 431 protected void closeRootAndMeta() throws IOException { 432 HBaseTestingUtility.closeRegionAndWAL(meta); 433 } 434 435 public static void assertByteEquals(byte[] expected, 436 byte[] actual) { 437 if (Bytes.compareTo(expected, actual) != 0) { 438 throw new AssertionFailedError("expected:<" + 439 Bytes.toString(expected) + "> but was:<" + 440 Bytes.toString(actual) + ">"); 441 } 442 } 443 444 public static void assertEquals(byte[] expected, 445 byte[] actual) { 446 if (Bytes.compareTo(expected, actual) != 0) { 447 throw new AssertionFailedError("expected:<" + 448 Bytes.toStringBinary(expected) + "> but was:<" + 449 Bytes.toStringBinary(actual) + ">"); 450 } 451 } 452}