Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.apache.hadoop.util.ToolRunner.run;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.HashMap;
028import java.util.HashSet;
029import java.util.List;
030import java.util.Map;
031import java.util.Objects;
032import java.util.Optional;
033import java.util.Set;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.fs.FileStatus;
036import org.apache.hadoop.fs.FileSystem;
037import org.apache.hadoop.fs.Path;
038import org.apache.hadoop.hbase.HBaseClassTestRule;
039import org.apache.hadoop.hbase.HBaseTestingUtility;
040import org.apache.hadoop.hbase.HConstants;
041import org.apache.hadoop.hbase.TableName;
042import org.apache.hadoop.hbase.client.Admin;
043import org.apache.hadoop.hbase.client.RegionInfo;
044import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
045import org.apache.hadoop.hbase.testclassification.LargeTests;
046import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.util.CommonFSUtils;
049import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
050import org.junit.After;
051import org.junit.AfterClass;
052import org.junit.Before;
053import org.junit.BeforeClass;
054import org.junit.ClassRule;
055import org.junit.Ignore;
056import org.junit.Rule;
057import org.junit.Test;
058import org.junit.experimental.categories.Category;
059import org.junit.rules.TestName;
060import org.slf4j.Logger;
061import org.slf4j.LoggerFactory;
062
063import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
064import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
065
066/**
067 * Test Export Snapshot Tool
068 */
069@Ignore // HBASE-24493
070@Category({ VerySlowMapReduceTests.class, LargeTests.class })
071public class TestExportSnapshot {
072
073  @ClassRule
074  public static final HBaseClassTestRule CLASS_RULE =
075    HBaseClassTestRule.forClass(TestExportSnapshot.class);
076
077  private static final Logger LOG = LoggerFactory.getLogger(TestExportSnapshot.class);
078
079  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
080
081  protected final static byte[] FAMILY = Bytes.toBytes("cf");
082
083  @Rule
084  public final TestName testName = new TestName();
085
086  protected TableName tableName;
087  private byte[] emptySnapshotName;
088  private byte[] snapshotName;
089  private int tableNumFiles;
090  private Admin admin;
091
092  public static void setUpBaseConf(Configuration conf) {
093    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
094    conf.setInt("hbase.regionserver.msginterval", 100);
095    // If a single node has enough failures (default 3), resource manager will blacklist it.
096    // With only 2 nodes and tests injecting faults, we don't want that.
097    conf.setInt("mapreduce.job.maxtaskfailures.per.tracker", 100);
098  }
099
100  @BeforeClass
101  public static void setUpBeforeClass() throws Exception {
102    setUpBaseConf(TEST_UTIL.getConfiguration());
103    TEST_UTIL.startMiniCluster(1);
104    TEST_UTIL.startMiniMapReduceCluster();
105  }
106
107  @AfterClass
108  public static void tearDownAfterClass() throws Exception {
109    TEST_UTIL.shutdownMiniMapReduceCluster();
110    TEST_UTIL.shutdownMiniCluster();
111  }
112
113  /**
114   * Create a table and take a snapshot of the table used by the export test.
115   */
116  @Before
117  public void setUp() throws Exception {
118    this.admin = TEST_UTIL.getAdmin();
119
120    tableName = TableName.valueOf("testtb-" + testName.getMethodName());
121    snapshotName = Bytes.toBytes("snaptb0-" + testName.getMethodName());
122    emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + testName.getMethodName());
123
124    // create Table
125    createTable(this.tableName);
126
127    // Take an empty snapshot
128    admin.snapshot(emptySnapshotName, tableName);
129
130    // Add some rows
131    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
132    tableNumFiles = admin.getTableRegions(tableName).size();
133
134    // take a snapshot
135    admin.snapshot(snapshotName, tableName);
136  }
137
138  protected void createTable(TableName tableName) throws Exception {
139    SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY);
140  }
141
142  protected interface RegionPredicate {
143    boolean evaluate(final RegionInfo regionInfo);
144  }
145
146  protected RegionPredicate getBypassRegionPredicate() {
147    return null;
148  }
149
150  @After
151  public void tearDown() throws Exception {
152    TEST_UTIL.deleteTable(tableName);
153    SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin());
154    SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
155  }
156
157  /**
158   * Verify if exported snapshot and copied files matches the original one.
159   */
160  @Test
161  public void testExportFileSystemState() throws Exception {
162    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
163  }
164
165  @Test
166  public void testExportFileSystemStateWithSkipTmp() throws Exception {
167    TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
168    try {
169      testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
170    } finally {
171      TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false);
172    }
173  }
174
175  @Test
176  public void testEmptyExportFileSystemState() throws Exception {
177    testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0);
178  }
179
180  @Test
181  public void testConsecutiveExports() throws Exception {
182    Path copyDir = getLocalDestinationDir(TEST_UTIL);
183    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false);
184    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true);
185    removeExportDir(copyDir);
186  }
187
188  @Test
189  public void testExportWithTargetName() throws Exception {
190    final byte[] targetName = Bytes.toBytes("testExportWithTargetName");
191    testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles);
192  }
193
194  @Test
195  public void testExportWithResetTtl() throws Exception {
196    String name = "testExportWithResetTtl";
197    TableName tableName = TableName.valueOf(name);
198    String snapshotNameStr = "snaptb-" + name;
199    byte[] snapshotName = Bytes.toBytes(snapshotNameStr);
200    Long ttl = 100000L;
201
202    try {
203      // create Table
204      createTable(tableName);
205      SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
206      int tableNumFiles = admin.getRegions(tableName).size();
207      // take a snapshot with TTL
208      Map<String, Object> props = new HashMap<>();
209      props.put("TTL", ttl);
210      admin.snapshot(snapshotNameStr, tableName, props);
211      Optional<Long> ttlOpt =
212        admin.listSnapshots().stream().filter(s -> s.getName().equals(snapshotNameStr))
213          .map(org.apache.hadoop.hbase.client.SnapshotDescription::getTtl).findAny();
214      assertTrue(ttlOpt.isPresent());
215      assertEquals(ttl, ttlOpt.get());
216
217      testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles,
218        getHdfsDestinationDir(), false, true);
219    } catch (Exception e) {
220      throw e;
221    } finally {
222      TEST_UTIL.deleteTable(tableName);
223    }
224  }
225
226  private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
227    final byte[] targetName, int filesExpected) throws Exception {
228    testExportFileSystemState(tableName, snapshotName, targetName, filesExpected,
229      getHdfsDestinationDir(), false);
230  }
231
232  protected void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
233    final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite) throws Exception {
234    testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir,
235      overwrite, false);
236  }
237
238  protected void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
239    final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite, boolean resetTtl)
240    throws Exception {
241    testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName,
242      filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite, resetTtl,
243      getBypassRegionPredicate(), true);
244  }
245
246  /**
247   * Creates destination directory, runs ExportSnapshot() tool, and runs some verifications.
248   */
249  protected static void testExportFileSystemState(final Configuration conf,
250    final TableName tableName, final byte[] snapshotName, final byte[] targetName,
251    final int filesExpected, final Path srcDir, Path rawTgtDir, final boolean overwrite,
252    final boolean resetTtl, final RegionPredicate bypassregionPredicate, boolean success)
253    throws Exception {
254    FileSystem tgtFs = rawTgtDir.getFileSystem(conf);
255    FileSystem srcFs = srcDir.getFileSystem(conf);
256    Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory());
257    LOG.info("tgtFsUri={}, tgtDir={}, rawTgtDir={}, srcFsUri={}, srcDir={}", tgtFs.getUri(), tgtDir,
258      rawTgtDir, srcFs.getUri(), srcDir);
259    List<String> opts = new ArrayList<>();
260    opts.add("--snapshot");
261    opts.add(Bytes.toString(snapshotName));
262    opts.add("--copy-to");
263    opts.add(tgtDir.toString());
264    if (targetName != snapshotName) {
265      opts.add("--target");
266      opts.add(Bytes.toString(targetName));
267    }
268    if (overwrite) {
269      opts.add("--overwrite");
270    }
271    if (resetTtl) {
272      opts.add("--reset-ttl");
273    }
274
275    // Export Snapshot
276    int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()]));
277    assertEquals("success " + success + ", res=" + res, success ? 0 : 1, res);
278    if (!success) {
279      final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName));
280      assertFalse(tgtDir.toString() + " " + targetDir.toString(),
281        tgtFs.exists(new Path(tgtDir, targetDir)));
282      return;
283    }
284    LOG.info("Exported snapshot");
285
286    // Verify File-System state
287    FileStatus[] rootFiles = tgtFs.listStatus(tgtDir);
288    assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length);
289    for (FileStatus fileStatus : rootFiles) {
290      String name = fileStatus.getPath().getName();
291      assertTrue(fileStatus.toString(), fileStatus.isDirectory());
292      assertTrue(name.toString(), name.equals(HConstants.SNAPSHOT_DIR_NAME)
293        || name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY));
294    }
295    LOG.info("Verified filesystem state");
296
297    // Compare the snapshot metadata and verify the hfiles
298    final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName));
299    final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName));
300    verifySnapshotDir(srcFs, new Path(srcDir, snapshotDir), tgtFs, new Path(tgtDir, targetDir));
301    Set<String> snapshotFiles = verifySnapshot(conf, tgtFs, tgtDir, tableName,
302      Bytes.toString(targetName), resetTtl, bypassregionPredicate);
303    assertEquals(filesExpected, snapshotFiles.size());
304  }
305
306  /*
307   * verify if the snapshot folder on file-system 1 match the one on file-system 2
308   */
309  protected static void verifySnapshotDir(final FileSystem fs1, final Path root1,
310    final FileSystem fs2, final Path root2) throws IOException {
311    assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2));
312  }
313
314  /*
315   * Verify if the files exists
316   */
317  protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs,
318    final Path rootDir, final TableName tableName, final String snapshotName,
319    final boolean resetTtl, final RegionPredicate bypassregionPredicate) throws IOException {
320    final Path exportedSnapshot =
321      new Path(rootDir, new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
322    final Set<String> snapshotFiles = new HashSet<>();
323    final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
324    SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot,
325      new SnapshotReferenceUtil.SnapshotVisitor() {
326        @Override
327        public void storeFile(final RegionInfo regionInfo, final String family,
328          final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
329          if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) {
330            return;
331          }
332
333          String hfile = storeFile.getName();
334          snapshotFiles.add(hfile);
335          if (!storeFile.hasReference()) {
336            verifyNonEmptyFile(new Path(exportedArchive,
337              new Path(CommonFSUtils.getTableDir(new Path("./"), tableName),
338                new Path(regionInfo.getEncodedName(), new Path(family, hfile)))));
339          }
340        }
341
342        private void verifyNonEmptyFile(final Path path) throws IOException {
343          assertTrue(path + " should exists", fs.exists(path));
344          assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
345        }
346      });
347
348    // Verify Snapshot description
349    SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
350    assertTrue(desc.getName().equals(snapshotName));
351    assertTrue(desc.getTable().equals(tableName.getNameAsString()));
352    if (resetTtl) {
353      assertEquals(HConstants.DEFAULT_SNAPSHOT_TTL, desc.getTtl());
354    }
355    return snapshotFiles;
356  }
357
358  private static Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)
359    throws IOException {
360    Set<String> files = new HashSet<>();
361    LOG.debug("List files in {} in root {} at {}", fs, root, dir);
362    int rootPrefix = root.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString().length();
363    FileStatus[] list = CommonFSUtils.listStatus(fs, dir);
364    if (list != null) {
365      for (FileStatus fstat : list) {
366        LOG.debug(Objects.toString(fstat.getPath()));
367        if (fstat.isDirectory()) {
368          files.addAll(listFiles(fs, root, fstat.getPath()));
369        } else {
370          files.add(fstat.getPath().makeQualified(fs).toString().substring(rootPrefix));
371        }
372      }
373    }
374    return files;
375  }
376
377  private Path getHdfsDestinationDir() {
378    Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
379    Path path =
380      new Path(new Path(rootDir, "export-test"), "export-" + EnvironmentEdgeManager.currentTime());
381    LOG.info("HDFS export destination path: " + path);
382    return path;
383  }
384
385  static Path getLocalDestinationDir(HBaseTestingUtility htu) {
386    Path path = htu.getDataTestDir("local-export-" + EnvironmentEdgeManager.currentTime());
387    try {
388      FileSystem fs = FileSystem.getLocal(htu.getConfiguration());
389      LOG.info("Local export destination path: " + path);
390      return path.makeQualified(fs.getUri(), fs.getWorkingDirectory());
391    } catch (IOException ioe) {
392      throw new RuntimeException(ioe);
393    }
394  }
395
396  private static void removeExportDir(final Path path) throws IOException {
397    FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
398    fs.delete(path, true);
399  }
400}