001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.snapshot; 019 020import static org.apache.hadoop.util.ToolRunner.run; 021import static org.junit.Assert.assertEquals; 022import static org.junit.Assert.assertFalse; 023import static org.junit.Assert.assertTrue; 024 025import java.io.IOException; 026import java.util.ArrayList; 027import java.util.HashMap; 028import java.util.HashSet; 029import java.util.List; 030import java.util.Map; 031import java.util.Objects; 032import java.util.Optional; 033import java.util.Set; 034import java.util.stream.Collectors; 035import org.apache.hadoop.conf.Configuration; 036import org.apache.hadoop.fs.FileStatus; 037import org.apache.hadoop.fs.FileSystem; 038import org.apache.hadoop.fs.Path; 039import org.apache.hadoop.hbase.HBaseClassTestRule; 040import org.apache.hadoop.hbase.HBaseTestingUtil; 041import org.apache.hadoop.hbase.HConstants; 042import org.apache.hadoop.hbase.TableName; 043import org.apache.hadoop.hbase.client.Admin; 044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 045import org.apache.hadoop.hbase.client.Put; 046import org.apache.hadoop.hbase.client.RegionInfo; 047import org.apache.hadoop.hbase.client.Table; 048import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 049import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; 050import org.apache.hadoop.hbase.regionserver.StoreFileInfo; 051import org.apache.hadoop.hbase.testclassification.LargeTests; 052import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests; 053import org.apache.hadoop.hbase.util.Bytes; 054import org.apache.hadoop.hbase.util.CommonFSUtils; 055import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 056import org.apache.hadoop.hbase.util.Pair; 057import org.junit.After; 058import org.junit.AfterClass; 059import org.junit.Before; 060import org.junit.BeforeClass; 061import org.junit.ClassRule; 062import org.junit.Ignore; 063import org.junit.Rule; 064import org.junit.Test; 065import org.junit.experimental.categories.Category; 066import org.junit.rules.TestName; 067import org.slf4j.Logger; 068import org.slf4j.LoggerFactory; 069 070import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 071 072import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; 073import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; 074 075/** 076 * Test Export Snapshot Tool 077 */ 078@Ignore // HBASE-24493 079@Category({ VerySlowMapReduceTests.class, LargeTests.class }) 080public class TestExportSnapshot { 081 082 @ClassRule 083 public static final HBaseClassTestRule CLASS_RULE = 084 HBaseClassTestRule.forClass(TestExportSnapshot.class); 085 086 private static final Logger LOG = LoggerFactory.getLogger(TestExportSnapshot.class); 087 088 protected final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); 089 090 protected final static byte[] FAMILY = Bytes.toBytes("cf"); 091 092 @Rule 093 public final TestName testName = new TestName(); 094 095 protected TableName tableName; 096 private String emptySnapshotName; 097 private String snapshotName; 098 private int tableNumFiles; 099 private Admin admin; 100 101 public static void setUpBaseConf(Configuration conf) { 102 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); 103 conf.setInt("hbase.regionserver.msginterval", 100); 104 // If a single node has enough failures (default 3), resource manager will blacklist it. 105 // With only 2 nodes and tests injecting faults, we don't want that. 106 conf.setInt("mapreduce.job.maxtaskfailures.per.tracker", 100); 107 } 108 109 @BeforeClass 110 public static void setUpBeforeClass() throws Exception { 111 setUpBaseConf(TEST_UTIL.getConfiguration()); 112 TEST_UTIL.startMiniCluster(1); 113 TEST_UTIL.startMiniMapReduceCluster(); 114 } 115 116 @AfterClass 117 public static void tearDownAfterClass() throws Exception { 118 TEST_UTIL.shutdownMiniMapReduceCluster(); 119 TEST_UTIL.shutdownMiniCluster(); 120 } 121 122 /** 123 * Create a table and take a snapshot of the table used by the export test. 124 */ 125 @Before 126 public void setUp() throws Exception { 127 this.admin = TEST_UTIL.getAdmin(); 128 129 tableName = TableName.valueOf("testtb-" + testName.getMethodName()); 130 snapshotName = "snaptb0-" + testName.getMethodName(); 131 emptySnapshotName = "emptySnaptb0-" + testName.getMethodName(); 132 133 // create Table 134 createTable(this.tableName); 135 136 // Take an empty snapshot 137 admin.snapshot(emptySnapshotName, tableName); 138 139 // Add some rows 140 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); 141 tableNumFiles = admin.getRegions(tableName).size(); 142 143 // take a snapshot 144 admin.snapshot(snapshotName, tableName); 145 } 146 147 protected void createTable(TableName tableName) throws Exception { 148 SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY); 149 } 150 151 protected interface RegionPredicate { 152 boolean evaluate(final RegionInfo regionInfo); 153 } 154 155 protected RegionPredicate getBypassRegionPredicate() { 156 return null; 157 } 158 159 @After 160 public void tearDown() throws Exception { 161 TEST_UTIL.deleteTable(tableName); 162 SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin()); 163 SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL); 164 } 165 166 /** 167 * Verify if exported snapshot and copied files matches the original one. 168 */ 169 @Test 170 public void testExportFileSystemState() throws Exception { 171 testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); 172 } 173 174 @Test 175 public void testExportFileSystemStateWithMergeRegion() throws Exception { 176 // disable compaction 177 admin.compactionSwitch(false, 178 admin.getRegionServers().stream().map(a -> a.getServerName()).collect(Collectors.toList())); 179 // create Table 180 TableName tableName0 = TableName.valueOf("testtb-" + testName.getMethodName() + "-1"); 181 String snapshotName0 = "snaptb0-" + testName.getMethodName() + "-1"; 182 admin.createTable( 183 TableDescriptorBuilder.newBuilder(tableName0) 184 .setColumnFamilies( 185 Lists.newArrayList(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build())) 186 .build(), 187 new byte[][] { Bytes.toBytes("2") }); 188 // put some data 189 try (Table table = admin.getConnection().getTable(tableName0)) { 190 table.put(new Put(Bytes.toBytes("1")).addColumn(FAMILY, null, Bytes.toBytes("1"))); 191 table.put(new Put(Bytes.toBytes("2")).addColumn(FAMILY, null, Bytes.toBytes("2"))); 192 } 193 List<RegionInfo> regions = admin.getRegions(tableName0); 194 assertEquals(2, regions.size()); 195 tableNumFiles = regions.size(); 196 // merge region 197 admin.mergeRegionsAsync(new byte[][] { regions.get(0).getEncodedNameAsBytes(), 198 regions.get(1).getEncodedNameAsBytes() }, true).get(); 199 // take a snapshot 200 admin.snapshot(snapshotName0, tableName0); 201 // export snapshot and verify 202 testExportFileSystemState(tableName0, snapshotName0, snapshotName0, tableNumFiles); 203 // delete table 204 TEST_UTIL.deleteTable(tableName0); 205 } 206 207 @Test 208 public void testExportFileSystemStateWithSkipTmp() throws Exception { 209 TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true); 210 try { 211 testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); 212 } finally { 213 TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false); 214 } 215 } 216 217 @Test 218 public void testEmptyExportFileSystemState() throws Exception { 219 testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0); 220 } 221 222 @Test 223 public void testConsecutiveExports() throws Exception { 224 Path copyDir = getLocalDestinationDir(TEST_UTIL); 225 testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false); 226 testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true); 227 removeExportDir(copyDir); 228 } 229 230 @Test 231 public void testExportWithTargetName() throws Exception { 232 final String targetName = "testExportWithTargetName"; 233 testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles); 234 } 235 236 @Test 237 public void testExportWithResetTtl() throws Exception { 238 String name = "testExportWithResetTtl"; 239 TableName tableName = TableName.valueOf(name); 240 String snapshotName = "snaptb-" + name; 241 Long ttl = 100000L; 242 243 try { 244 // create Table 245 createTable(tableName); 246 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); 247 int tableNumFiles = admin.getRegions(tableName).size(); 248 // take a snapshot with TTL 249 Map<String, Object> props = new HashMap<>(); 250 props.put("TTL", ttl); 251 admin.snapshot(snapshotName, tableName, props); 252 Optional<Long> ttlOpt = 253 admin.listSnapshots().stream().filter(s -> s.getName().equals(snapshotName)) 254 .map(org.apache.hadoop.hbase.client.SnapshotDescription::getTtl).findAny(); 255 assertTrue(ttlOpt.isPresent()); 256 assertEquals(ttl, ttlOpt.get()); 257 258 testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, 259 getHdfsDestinationDir(), false, true); 260 } catch (Exception e) { 261 throw e; 262 } finally { 263 TEST_UTIL.deleteTable(tableName); 264 } 265 } 266 267 private void testExportFileSystemState(final TableName tableName, final String snapshotName, 268 final String targetName, int filesExpected) throws Exception { 269 testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, 270 getHdfsDestinationDir(), false); 271 } 272 273 protected void testExportFileSystemState(final TableName tableName, final String snapshotName, 274 final String targetName, int filesExpected, Path copyDir, boolean overwrite) throws Exception { 275 testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir, 276 overwrite, false); 277 } 278 279 protected void testExportFileSystemState(final TableName tableName, final String snapshotName, 280 final String targetName, int filesExpected, Path copyDir, boolean overwrite, boolean resetTtl) 281 throws Exception { 282 testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName, 283 filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite, resetTtl, 284 getBypassRegionPredicate(), true); 285 } 286 287 /** 288 * Creates destination directory, runs ExportSnapshot() tool, and runs some verifications. 289 */ 290 protected static void testExportFileSystemState(final Configuration conf, 291 final TableName tableName, final String snapshotName, final String targetName, 292 final int filesExpected, final Path srcDir, Path rawTgtDir, final boolean overwrite, 293 final boolean resetTtl, final RegionPredicate bypassregionPredicate, boolean success) 294 throws Exception { 295 FileSystem tgtFs = rawTgtDir.getFileSystem(conf); 296 FileSystem srcFs = srcDir.getFileSystem(conf); 297 Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory()); 298 LOG.info("tgtFsUri={}, tgtDir={}, rawTgtDir={}, srcFsUri={}, srcDir={}", tgtFs.getUri(), tgtDir, 299 rawTgtDir, srcFs.getUri(), srcDir); 300 List<String> opts = new ArrayList<>(); 301 opts.add("--snapshot"); 302 opts.add(snapshotName); 303 opts.add("--copy-to"); 304 opts.add(tgtDir.toString()); 305 if (!targetName.equals(snapshotName)) { 306 opts.add("--target"); 307 opts.add(targetName); 308 } 309 if (overwrite) { 310 opts.add("--overwrite"); 311 } 312 if (resetTtl) { 313 opts.add("--reset-ttl"); 314 } 315 316 // Export Snapshot 317 int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()])); 318 assertEquals("success " + success + ", res=" + res, success ? 0 : 1, res); 319 if (!success) { 320 final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName); 321 assertFalse(tgtDir.toString() + " " + targetDir.toString(), 322 tgtFs.exists(new Path(tgtDir, targetDir))); 323 return; 324 } 325 LOG.info("Exported snapshot"); 326 327 // Verify File-System state 328 FileStatus[] rootFiles = tgtFs.listStatus(tgtDir); 329 assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length); 330 for (FileStatus fileStatus : rootFiles) { 331 String name = fileStatus.getPath().getName(); 332 assertTrue(fileStatus.toString(), fileStatus.isDirectory()); 333 assertTrue(name.toString(), name.equals(HConstants.SNAPSHOT_DIR_NAME) 334 || name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY)); 335 } 336 LOG.info("Verified filesystem state"); 337 338 // Compare the snapshot metadata and verify the hfiles 339 final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName); 340 final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName); 341 verifySnapshotDir(srcFs, new Path(srcDir, snapshotDir), tgtFs, new Path(tgtDir, targetDir)); 342 Set<String> snapshotFiles = 343 verifySnapshot(conf, tgtFs, tgtDir, tableName, targetName, resetTtl, bypassregionPredicate); 344 assertEquals(filesExpected, snapshotFiles.size()); 345 } 346 347 /* 348 * verify if the snapshot folder on file-system 1 match the one on file-system 2 349 */ 350 protected static void verifySnapshotDir(final FileSystem fs1, final Path root1, 351 final FileSystem fs2, final Path root2) throws IOException { 352 assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2)); 353 } 354 355 /* 356 * Verify if the files exists 357 */ 358 protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs, 359 final Path rootDir, final TableName tableName, final String snapshotName, 360 final boolean resetTtl, final RegionPredicate bypassregionPredicate) throws IOException { 361 final Path exportedSnapshot = 362 new Path(rootDir, new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName)); 363 final Set<String> snapshotFiles = new HashSet<>(); 364 final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); 365 SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot, 366 new SnapshotReferenceUtil.SnapshotVisitor() { 367 @Override 368 public void storeFile(final RegionInfo regionInfo, final String family, 369 final SnapshotRegionManifest.StoreFile storeFile) throws IOException { 370 if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) { 371 return; 372 } 373 374 if (!storeFile.hasReference() && !StoreFileInfo.isReference(storeFile.getName())) { 375 String hfile = storeFile.getName(); 376 snapshotFiles.add(hfile); 377 verifyNonEmptyFile(new Path(exportedArchive, 378 new Path(CommonFSUtils.getTableDir(new Path("./"), tableName), 379 new Path(regionInfo.getEncodedName(), new Path(family, hfile))))); 380 } else { 381 Pair<String, String> referredToRegionAndFile = 382 StoreFileInfo.getReferredToRegionAndFile(storeFile.getName()); 383 String region = referredToRegionAndFile.getFirst(); 384 String hfile = referredToRegionAndFile.getSecond(); 385 snapshotFiles.add(hfile); 386 verifyNonEmptyFile(new Path(exportedArchive, 387 new Path(CommonFSUtils.getTableDir(new Path("./"), tableName), 388 new Path(region, new Path(family, hfile))))); 389 } 390 } 391 392 private void verifyNonEmptyFile(final Path path) throws IOException { 393 assertTrue(path + " should exists", fs.exists(path)); 394 assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0); 395 } 396 }); 397 398 // Verify Snapshot description 399 SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot); 400 assertTrue(desc.getName().equals(snapshotName)); 401 assertTrue(desc.getTable().equals(tableName.getNameAsString())); 402 if (resetTtl) { 403 assertEquals(HConstants.DEFAULT_SNAPSHOT_TTL, desc.getTtl()); 404 } 405 return snapshotFiles; 406 } 407 408 private static Set<String> listFiles(final FileSystem fs, final Path root, final Path dir) 409 throws IOException { 410 Set<String> files = new HashSet<>(); 411 LOG.debug("List files in {} in root {} at {}", fs, root, dir); 412 int rootPrefix = root.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString().length(); 413 FileStatus[] list = CommonFSUtils.listStatus(fs, dir); 414 if (list != null) { 415 for (FileStatus fstat : list) { 416 LOG.debug(Objects.toString(fstat.getPath())); 417 if (fstat.isDirectory()) { 418 files.addAll(listFiles(fs, root, fstat.getPath())); 419 } else { 420 files.add(fstat.getPath().makeQualified(fs).toString().substring(rootPrefix)); 421 } 422 } 423 } 424 return files; 425 } 426 427 private Path getHdfsDestinationDir() { 428 Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); 429 Path path = 430 new Path(new Path(rootDir, "export-test"), "export-" + EnvironmentEdgeManager.currentTime()); 431 LOG.info("HDFS export destination path: " + path); 432 return path; 433 } 434 435 static Path getLocalDestinationDir(HBaseTestingUtil htu) { 436 Path path = htu.getDataTestDir("local-export-" + EnvironmentEdgeManager.currentTime()); 437 try { 438 FileSystem fs = FileSystem.getLocal(htu.getConfiguration()); 439 LOG.info("Local export destination path: " + path); 440 return path.makeQualified(fs.getUri(), fs.getWorkingDirectory()); 441 } catch (IOException ioe) { 442 throw new RuntimeException(ioe); 443 } 444 } 445 446 private static void removeExportDir(final Path path) throws IOException { 447 FileSystem fs = FileSystem.get(path.toUri(), new Configuration()); 448 fs.delete(path, true); 449 } 450}