001/** 002 * 003 * Licensed to the Apache Software Foundation (ASF) under one 004 * or more contributor license agreements. See the NOTICE file 005 * distributed with this work for additional information 006 * regarding copyright ownership. The ASF licenses this file 007 * to you under the Apache License, Version 2.0 (the 008 * "License"); you may not use this file except in compliance 009 * with the License. You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 */ 019package org.apache.hadoop.hbase; 020 021import java.io.IOException; 022import java.nio.charset.StandardCharsets; 023import java.util.NavigableMap; 024 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.fs.FileSystem; 027import org.apache.hadoop.fs.Path; 028import org.apache.hadoop.hbase.client.Durability; 029import org.apache.hadoop.hbase.client.Get; 030import org.apache.hadoop.hbase.client.Put; 031import org.apache.hadoop.hbase.client.Result; 032import org.apache.hadoop.hbase.client.Table; 033import org.apache.hadoop.hbase.log.HBaseMarkers; 034import org.apache.hadoop.hbase.regionserver.HRegion; 035import org.apache.hadoop.hbase.regionserver.Region; 036import org.apache.hadoop.hbase.regionserver.RegionAsTable; 037import org.apache.hadoop.hbase.util.Bytes; 038import org.apache.hadoop.hbase.util.FSTableDescriptors; 039import org.apache.hadoop.hbase.util.FSUtils; 040import org.apache.hadoop.hdfs.MiniDFSCluster; 041import org.slf4j.Logger; 042import org.slf4j.LoggerFactory; 043 044import junit.framework.AssertionFailedError; 045import junit.framework.TestCase; 046 047/** 048 * Abstract HBase test class. Initializes a few things that can come in handly 049 * like an HBaseConfiguration and filesystem. 050 * @deprecated since 2.0.0 and will be removed in 3.0.0. Write junit4 unit tests using 051 * {@link HBaseTestingUtility}. 052 * @see HBaseTestingUtility 053 * @see <a href="https://issues.apache.org/jira/browse/HBASE-11912">HBASE-11912</a> 054 */ 055@Deprecated 056public abstract class HBaseTestCase extends TestCase { 057 private static final Logger LOG = LoggerFactory.getLogger(HBaseTestCase.class); 058 059 protected final static byte [] fam1 = Bytes.toBytes("colfamily11"); 060 protected final static byte [] fam2 = Bytes.toBytes("colfamily21"); 061 protected final static byte [] fam3 = Bytes.toBytes("colfamily31"); 062 063 protected static final byte [][] COLUMNS = {fam1, fam2, fam3}; 064 065 private boolean localfs = false; 066 protected static Path testDir = null; 067 protected FileSystem fs = null; 068 protected HRegion meta = null; 069 protected static final char FIRST_CHAR = 'a'; 070 protected static final char LAST_CHAR = 'z'; 071 protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|"; 072 protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR}; 073 protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET); 074 protected static final int MAXVERSIONS = 3; 075 076 protected final HBaseTestingUtility testUtil = new HBaseTestingUtility(); 077 078 public volatile Configuration conf = testUtil.getConfiguration(); 079 public final FSTableDescriptors fsTableDescriptors; 080 { 081 try { 082 fsTableDescriptors = new FSTableDescriptors(conf); 083 } catch (IOException e) { 084 throw new RuntimeException("Failed to init descriptors", e); 085 } 086 } 087 088 /** constructor */ 089 public HBaseTestCase() { 090 super(); 091 } 092 093 /** 094 * @param name 095 */ 096 public HBaseTestCase(String name) { 097 super(name); 098 } 099 100 /** 101 * Note that this method must be called after the mini hdfs cluster has 102 * started or we end up with a local file system. 103 */ 104 @Override 105 protected void setUp() throws Exception { 106 super.setUp(); 107 localfs = 108 (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0); 109 110 if (fs == null) { 111 this.fs = FileSystem.get(conf); 112 } 113 try { 114 if (localfs) { 115 testDir = getUnitTestdir(getName()); 116 if (fs.exists(testDir)) { 117 fs.delete(testDir, true); 118 } 119 } else { 120 testDir = FSUtils.getRootDir(conf); 121 } 122 } catch (Exception e) { 123 LOG.error(HBaseMarkers.FATAL, "error during setup", e); 124 throw e; 125 } 126 } 127 128 @Override 129 protected void tearDown() throws Exception { 130 try { 131 if (localfs) { 132 if (this.fs.exists(testDir)) { 133 this.fs.delete(testDir, true); 134 } 135 } 136 } catch (Exception e) { 137 LOG.error(HBaseMarkers.FATAL, "error during tear down", e); 138 } 139 super.tearDown(); 140 } 141 142 /** 143 * @see HBaseTestingUtility#getBaseTestDir 144 * @param testName 145 * @return directory to use for this test 146 */ 147 protected Path getUnitTestdir(String testName) { 148 return testUtil.getDataTestDir(testName); 149 } 150 151 /** 152 * You must call close on the returned region and then close on the log file it created. Do 153 * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} to close both the region and the WAL. 154 * @param desc 155 * @param startKey 156 * @param endKey 157 * @return An {@link HRegion} 158 * @throws IOException 159 */ 160 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey, 161 byte [] endKey) 162 throws IOException { 163 return createNewHRegion(desc, startKey, endKey, this.conf); 164 } 165 166 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey, 167 byte [] endKey, Configuration conf) 168 throws IOException { 169 HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey); 170 return HBaseTestingUtility.createRegionAndWAL(hri, testDir, conf, desc); 171 } 172 173 protected HRegion openClosedRegion(final HRegion closedRegion) 174 throws IOException { 175 return HRegion.openHRegion(closedRegion, null); 176 } 177 178 /** 179 * Create a table of name <code>name</code> with {@link COLUMNS} for 180 * families. 181 * @param name Name to give table. 182 * @return Column descriptor. 183 */ 184 protected HTableDescriptor createTableDescriptor(final String name) { 185 return createTableDescriptor(name, MAXVERSIONS); 186 } 187 188 /** 189 * Create a table of name <code>name</code> with {@link COLUMNS} for 190 * families. 191 * @param name Name to give table. 192 * @param versions How many versions to allow per column. 193 * @return Column descriptor. 194 */ 195 protected HTableDescriptor createTableDescriptor(final String name, 196 final int versions) { 197 return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS, 198 versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED); 199 } 200 201 /** 202 * Create a table of name <code>name</code> with {@link COLUMNS} for 203 * families. 204 * @param name Name to give table. 205 * @param versions How many versions to allow per column. 206 * @return Column descriptor. 207 */ 208 protected HTableDescriptor createTableDescriptor(final String name, 209 final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) { 210 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); 211 for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) { 212 htd.addFamily(new HColumnDescriptor(cfName) 213 .setMinVersions(minVersions) 214 .setMaxVersions(versions) 215 .setKeepDeletedCells(keepDeleted) 216 .setBlockCacheEnabled(false) 217 .setTimeToLive(ttl) 218 ); 219 } 220 return htd; 221 } 222 223 /** 224 * Add content to region <code>r</code> on the passed column 225 * <code>column</code>. 226 * Adds data of the from 'aaa', 'aab', etc where key and value are the same. 227 * @param r 228 * @param columnFamily 229 * @param column 230 * @throws IOException 231 * @return count of what we added. 232 */ 233 public static long addContent(final Region r, final byte [] columnFamily, final byte[] column) 234 throws IOException { 235 byte [] startKey = r.getRegionInfo().getStartKey(); 236 byte [] endKey = r.getRegionInfo().getEndKey(); 237 byte [] startKeyBytes = startKey; 238 if (startKeyBytes == null || startKeyBytes.length == 0) { 239 startKeyBytes = START_KEY_BYTES; 240 } 241 return addContent(new RegionAsTable(r), Bytes.toString(columnFamily), Bytes.toString(column), 242 startKeyBytes, endKey, -1); 243 } 244 245 public static long addContent(final Region r, final byte [] columnFamily) throws IOException { 246 return addContent(r, columnFamily, null); 247 } 248 249 /** 250 * Add content to region <code>r</code> on the passed column 251 * <code>column</code>. 252 * Adds data of the from 'aaa', 'aab', etc where key and value are the same. 253 * @throws IOException 254 * @return count of what we added. 255 */ 256 public static long addContent(final Table updater, 257 final String columnFamily) throws IOException { 258 return addContent(updater, columnFamily, START_KEY_BYTES, null); 259 } 260 261 public static long addContent(final Table updater, final String family, 262 final String column) throws IOException { 263 return addContent(updater, family, column, START_KEY_BYTES, null); 264 } 265 266 /** 267 * Add content to region <code>r</code> on the passed column 268 * <code>column</code>. 269 * Adds data of the from 'aaa', 'aab', etc where key and value are the same. 270 * @return count of what we added. 271 * @throws IOException 272 */ 273 public static long addContent(final Table updater, final String columnFamily, 274 final byte [] startKeyBytes, final byte [] endKey) 275 throws IOException { 276 return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1); 277 } 278 279 public static long addContent(final Table updater, final String family, String column, 280 final byte [] startKeyBytes, final byte [] endKey) throws IOException { 281 return addContent(updater, family, column, startKeyBytes, endKey, -1); 282 } 283 284 /** 285 * Add content to region <code>r</code> on the passed column 286 * <code>column</code>. 287 * Adds data of the from 'aaa', 'aab', etc where key and value are the same. 288 * @return count of what we added. 289 * @throws IOException 290 */ 291 public static long addContent(final Table updater, 292 final String columnFamily, 293 final String column, 294 final byte [] startKeyBytes, final byte [] endKey, final long ts) 295 throws IOException { 296 long count = 0; 297 // Add rows of three characters. The first character starts with the 298 // 'a' character and runs up to 'z'. Per first character, we run the 299 // second character over same range. And same for the third so rows 300 // (and values) look like this: 'aaa', 'aab', 'aac', etc. 301 char secondCharStart = (char)startKeyBytes[1]; 302 char thirdCharStart = (char)startKeyBytes[2]; 303 EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) { 304 for (char d = secondCharStart; d <= LAST_CHAR; d++) { 305 for (char e = thirdCharStart; e <= LAST_CHAR; e++) { 306 byte [] t = new byte [] {(byte)c, (byte)d, (byte)e}; 307 if (endKey != null && endKey.length > 0 308 && Bytes.compareTo(endKey, t) <= 0) { 309 break EXIT; 310 } 311 try { 312 Put put; 313 if(ts != -1) { 314 put = new Put(t, ts); 315 } else { 316 put = new Put(t); 317 } 318 try { 319 StringBuilder sb = new StringBuilder(); 320 if (column != null && column.contains(":")) { 321 sb.append(column); 322 } else { 323 if (columnFamily != null) { 324 sb.append(columnFamily); 325 if (!columnFamily.endsWith(":")) { 326 sb.append(":"); 327 } 328 if (column != null) { 329 sb.append(column); 330 } 331 } 332 } 333 byte[][] split = 334 CellUtil.parseColumn(Bytes.toBytes(sb.toString())); 335 if(split.length == 1) { 336 byte[] qualifier = new byte[0]; 337 put.addColumn(split[0], qualifier, t); 338 } else { 339 put.addColumn(split[0], split[1], t); 340 } 341 put.setDurability(Durability.SKIP_WAL); 342 updater.put(put); 343 count++; 344 } catch (RuntimeException ex) { 345 ex.printStackTrace(); 346 throw ex; 347 } catch (IOException ex) { 348 ex.printStackTrace(); 349 throw ex; 350 } 351 } catch (RuntimeException ex) { 352 ex.printStackTrace(); 353 throw ex; 354 } catch (IOException ex) { 355 ex.printStackTrace(); 356 throw ex; 357 } 358 } 359 // Set start character back to FIRST_CHAR after we've done first loop. 360 thirdCharStart = FIRST_CHAR; 361 } 362 secondCharStart = FIRST_CHAR; 363 } 364 return count; 365 } 366 367 protected void assertResultEquals(final HRegion region, final byte [] row, 368 final byte [] family, final byte [] qualifier, final long timestamp, 369 final byte [] value) throws IOException { 370 Get get = new Get(row); 371 get.setTimestamp(timestamp); 372 Result res = region.get(get); 373 NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map = 374 res.getMap(); 375 byte [] res_value = map.get(family).get(qualifier).get(timestamp); 376 377 if (value == null) { 378 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) + 379 " at timestamp " + timestamp, null, res_value); 380 } else { 381 if (res_value == null) { 382 fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) + 383 " at timestamp " + timestamp + "\" was expected to be \"" + 384 Bytes.toStringBinary(value) + " but was null"); 385 } 386 if (res_value != null) { 387 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) + 388 " at timestamp " + 389 timestamp, value, new String(res_value, StandardCharsets.UTF_8)); 390 } 391 } 392 } 393 394 /** 395 * Common method to close down a MiniDFSCluster and the associated file system 396 * 397 * @param cluster 398 */ 399 public static void shutdownDfs(MiniDFSCluster cluster) { 400 if (cluster != null) { 401 LOG.info("Shutting down Mini DFS "); 402 try { 403 cluster.shutdown(); 404 } catch (Exception e) { 405 /// Can get a java.lang.reflect.UndeclaredThrowableException thrown 406 // here because of an InterruptedException. Don't let exceptions in 407 // here be cause of test failure. 408 } 409 try { 410 FileSystem fs = cluster.getFileSystem(); 411 if (fs != null) { 412 LOG.info("Shutting down FileSystem"); 413 fs.close(); 414 } 415 FileSystem.closeAll(); 416 } catch (IOException e) { 417 LOG.error("error closing file system", e); 418 } 419 } 420 } 421 422 /** 423 * You must call {@link #closeRootAndMeta()} when done after calling this 424 * method. It does cleanup. 425 * @throws IOException 426 */ 427 protected void createMetaRegion() throws IOException { 428 FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(conf); 429 meta = HBaseTestingUtility.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO, testDir, 430 conf, fsTableDescriptors.get(TableName.META_TABLE_NAME)); 431 } 432 433 protected void closeRootAndMeta() throws IOException { 434 HBaseTestingUtility.closeRegionAndWAL(meta); 435 } 436 437 public static void assertByteEquals(byte[] expected, 438 byte[] actual) { 439 if (Bytes.compareTo(expected, actual) != 0) { 440 throw new AssertionFailedError("expected:<" + 441 Bytes.toString(expected) + "> but was:<" + 442 Bytes.toString(actual) + ">"); 443 } 444 } 445 446 public static void assertEquals(byte[] expected, 447 byte[] actual) { 448 if (Bytes.compareTo(expected, actual) != 0) { 449 throw new AssertionFailedError("expected:<" + 450 Bytes.toStringBinary(expected) + "> but was:<" + 451 Bytes.toStringBinary(actual) + ">"); 452 } 453 } 454}