001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.apache.hadoop.util.ToolRunner.run;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Objects;
030import java.util.Set;
031import org.apache.hadoop.conf.Configuration;
032import org.apache.hadoop.fs.FileStatus;
033import org.apache.hadoop.fs.FileSystem;
034import org.apache.hadoop.fs.Path;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.HBaseTestingUtility;
037import org.apache.hadoop.hbase.HConstants;
038import org.apache.hadoop.hbase.TableName;
039import org.apache.hadoop.hbase.client.Admin;
040import org.apache.hadoop.hbase.client.RegionInfo;
041import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
042import org.apache.hadoop.hbase.testclassification.LargeTests;
043import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.hbase.util.CommonFSUtils;
046import org.junit.After;
047import org.junit.AfterClass;
048import org.junit.Before;
049import org.junit.BeforeClass;
050import org.junit.ClassRule;
051import org.junit.Ignore;
052import org.junit.Rule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.junit.rules.TestName;
056import org.slf4j.Logger;
057import org.slf4j.LoggerFactory;
058
059import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
060import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
061
062/**
063 * Test Export Snapshot Tool
064 */
065@Ignore // HBASE-24493
066@Category({VerySlowMapReduceTests.class, LargeTests.class})
067public class TestExportSnapshot {
068
069  @ClassRule
070  public static final HBaseClassTestRule CLASS_RULE =
071      HBaseClassTestRule.forClass(TestExportSnapshot.class);
072
073  private static final Logger LOG = LoggerFactory.getLogger(TestExportSnapshot.class);
074
075  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
076
077  protected final static byte[] FAMILY = Bytes.toBytes("cf");
078
079  @Rule
080  public final TestName testName = new TestName();
081
082  protected TableName tableName;
083  private byte[] emptySnapshotName;
084  private byte[] snapshotName;
085  private int tableNumFiles;
086  private Admin admin;
087
088  public static void setUpBaseConf(Configuration conf) {
089    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
090    conf.setInt("hbase.regionserver.msginterval", 100);
091    // If a single node has enough failures (default 3), resource manager will blacklist it.
092    // With only 2 nodes and tests injecting faults, we don't want that.
093    conf.setInt("mapreduce.job.maxtaskfailures.per.tracker", 100);
094  }
095
096  @BeforeClass
097  public static void setUpBeforeClass() throws Exception {
098    setUpBaseConf(TEST_UTIL.getConfiguration());
099    TEST_UTIL.startMiniCluster(1);
100    TEST_UTIL.startMiniMapReduceCluster();
101  }
102
103  @AfterClass
104  public static void tearDownAfterClass() throws Exception {
105    TEST_UTIL.shutdownMiniMapReduceCluster();
106    TEST_UTIL.shutdownMiniCluster();
107  }
108
109  /**
110   * Create a table and take a snapshot of the table used by the export test.
111   */
112  @Before
113  public void setUp() throws Exception {
114    this.admin = TEST_UTIL.getAdmin();
115
116    tableName = TableName.valueOf("testtb-" + testName.getMethodName());
117    snapshotName = Bytes.toBytes("snaptb0-" + testName.getMethodName());
118    emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + testName.getMethodName());
119
120    // create Table
121    createTable(this.tableName);
122
123    // Take an empty snapshot
124    admin.snapshot(emptySnapshotName, tableName);
125
126    // Add some rows
127    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
128    tableNumFiles = admin.getTableRegions(tableName).size();
129
130    // take a snapshot
131    admin.snapshot(snapshotName, tableName);
132  }
133
134  protected void createTable(TableName tableName) throws Exception {
135    SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY);
136  }
137
138  protected interface RegionPredicate {
139    boolean evaluate(final RegionInfo regionInfo);
140  }
141
142  protected RegionPredicate getBypassRegionPredicate() {
143    return null;
144  }
145
146  @After
147  public void tearDown() throws Exception {
148    TEST_UTIL.deleteTable(tableName);
149    SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin());
150    SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
151  }
152
153  /**
154   * Verify if exported snapshot and copied files matches the original one.
155   */
156  @Test
157  public void testExportFileSystemState() throws Exception {
158    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
159  }
160
161  @Test
162  public void testExportFileSystemStateWithSkipTmp() throws Exception {
163    TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
164    try {
165      testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
166    } finally {
167      TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false);
168    }
169  }
170
171  @Test
172  public void testEmptyExportFileSystemState() throws Exception {
173    testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0);
174  }
175
176  @Test
177  public void testConsecutiveExports() throws Exception {
178    Path copyDir = getLocalDestinationDir(TEST_UTIL);
179    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false);
180    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true);
181    removeExportDir(copyDir);
182  }
183
184  @Test
185  public void testExportWithTargetName() throws Exception {
186    final byte[] targetName = Bytes.toBytes("testExportWithTargetName");
187    testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles);
188  }
189
190  private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
191      final byte[] targetName, int filesExpected) throws Exception {
192    testExportFileSystemState(tableName, snapshotName, targetName,
193      filesExpected, getHdfsDestinationDir(), false);
194  }
195
196  protected void testExportFileSystemState(final TableName tableName,
197      final byte[] snapshotName, final byte[] targetName, int filesExpected,
198      Path copyDir, boolean overwrite) throws Exception {
199    testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName,
200      filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir,
201      overwrite, getBypassRegionPredicate(), true);
202  }
203
204  /**
205   * Creates destination directory, runs ExportSnapshot() tool, and runs some verifications.
206   */
207  protected static void testExportFileSystemState(final Configuration conf,
208      final TableName tableName,
209      final byte[] snapshotName, final byte[] targetName, final int filesExpected,
210      final Path srcDir, Path rawTgtDir, final boolean overwrite,
211      final RegionPredicate bypassregionPredicate, boolean success) throws Exception {
212    FileSystem tgtFs = rawTgtDir.getFileSystem(conf);
213    FileSystem srcFs = srcDir.getFileSystem(conf);
214    Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory());
215    LOG.info("tgtFsUri={}, tgtDir={}, rawTgtDir={}, srcFsUri={}, srcDir={}",
216      tgtFs.getUri(), tgtDir, rawTgtDir, srcFs.getUri(), srcDir);
217    List<String> opts = new ArrayList<>();
218    opts.add("--snapshot");
219    opts.add(Bytes.toString(snapshotName));
220    opts.add("--copy-to");
221    opts.add(tgtDir.toString());
222    if (targetName != snapshotName) {
223      opts.add("--target");
224      opts.add(Bytes.toString(targetName));
225    }
226    if (overwrite) {
227      opts.add("--overwrite");
228    }
229
230    // Export Snapshot
231    int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()]));
232    assertEquals("success " + success + ", res=" + res, success ? 0 : 1, res);
233    if (!success) {
234      final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName));
235      assertFalse(tgtDir.toString() + " " + targetDir.toString(),
236        tgtFs.exists(new Path(tgtDir, targetDir)));
237      return;
238    }
239    LOG.info("Exported snapshot");
240
241    // Verify File-System state
242    FileStatus[] rootFiles = tgtFs.listStatus(tgtDir);
243    assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length);
244    for (FileStatus fileStatus: rootFiles) {
245      String name = fileStatus.getPath().getName();
246      assertTrue(fileStatus.toString(), fileStatus.isDirectory());
247      assertTrue(name.toString(), name.equals(HConstants.SNAPSHOT_DIR_NAME) ||
248        name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY));
249    }
250    LOG.info("Verified filesystem state");
251
252    // Compare the snapshot metadata and verify the hfiles
253    final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName));
254    final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName));
255    verifySnapshotDir(srcFs, new Path(srcDir, snapshotDir), tgtFs, new Path(tgtDir, targetDir));
256    Set<String> snapshotFiles = verifySnapshot(conf, tgtFs, tgtDir, tableName,
257      Bytes.toString(targetName), bypassregionPredicate);
258    assertEquals(filesExpected, snapshotFiles.size());
259  }
260
261  /*
262   * verify if the snapshot folder on file-system 1 match the one on file-system 2
263   */
264  protected static void verifySnapshotDir(final FileSystem fs1, final Path root1,
265      final FileSystem fs2, final Path root2) throws IOException {
266    assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2));
267  }
268
269  /*
270   * Verify if the files exists
271   */
272  protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs,
273      final Path rootDir, final TableName tableName, final String snapshotName,
274      final RegionPredicate bypassregionPredicate) throws IOException {
275    final Path exportedSnapshot = new Path(rootDir,
276      new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
277    final Set<String> snapshotFiles = new HashSet<>();
278    final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
279    SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot,
280          new SnapshotReferenceUtil.SnapshotVisitor() {
281        @Override
282        public void storeFile(final RegionInfo regionInfo, final String family,
283            final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
284          if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) {
285            return;
286          }
287
288          String hfile = storeFile.getName();
289          snapshotFiles.add(hfile);
290          if (!storeFile.hasReference()) {
291            verifyNonEmptyFile(new Path(exportedArchive,
292              new Path(CommonFSUtils.getTableDir(new Path("./"), tableName),
293                  new Path(regionInfo.getEncodedName(), new Path(family, hfile)))));
294          }
295        }
296
297        private void verifyNonEmptyFile(final Path path) throws IOException {
298          assertTrue(path + " should exists", fs.exists(path));
299          assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
300        }
301      });
302
303    // Verify Snapshot description
304    SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
305    assertTrue(desc.getName().equals(snapshotName));
306    assertTrue(desc.getTable().equals(tableName.getNameAsString()));
307    return snapshotFiles;
308  }
309
310  private static Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)
311      throws IOException {
312    Set<String> files = new HashSet<>();
313    LOG.debug("List files in {} in root {} at {}", fs, root, dir);
314    int rootPrefix = root.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString().length();
315    FileStatus[] list = CommonFSUtils.listStatus(fs, dir);
316    if (list != null) {
317      for (FileStatus fstat: list) {
318        LOG.debug(Objects.toString(fstat.getPath()));
319        if (fstat.isDirectory()) {
320          files.addAll(listFiles(fs, root, fstat.getPath()));
321        } else {
322          files.add(fstat.getPath().makeQualified(fs).toString().substring(rootPrefix));
323        }
324      }
325    }
326    return files;
327  }
328
329  private Path getHdfsDestinationDir() {
330    Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
331    Path path = new Path(new Path(rootDir, "export-test"), "export-" + System.currentTimeMillis());
332    LOG.info("HDFS export destination path: " + path);
333    return path;
334  }
335
336  static Path getLocalDestinationDir(HBaseTestingUtility htu) {
337    Path path = htu.getDataTestDir("local-export-" + System.currentTimeMillis());
338    try {
339      FileSystem fs = FileSystem.getLocal(htu.getConfiguration());
340      LOG.info("Local export destination path: " + path);
341      return path.makeQualified(fs.getUri(), fs.getWorkingDirectory());
342    } catch (IOException ioe) {
343      throw new RuntimeException(ioe);
344    }
345  }
346
347  private static void removeExportDir(final Path path) throws IOException {
348    FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
349    fs.delete(path, true);
350  }
351}