001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.apache.hadoop.util.ToolRunner.run;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Objects;
030import java.util.Set;
031import org.apache.hadoop.conf.Configuration;
032import org.apache.hadoop.fs.FileStatus;
033import org.apache.hadoop.fs.FileSystem;
034import org.apache.hadoop.fs.Path;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.HBaseTestingUtil;
037import org.apache.hadoop.hbase.HConstants;
038import org.apache.hadoop.hbase.TableName;
039import org.apache.hadoop.hbase.client.Admin;
040import org.apache.hadoop.hbase.client.RegionInfo;
041import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
042import org.apache.hadoop.hbase.testclassification.LargeTests;
043import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.hbase.util.CommonFSUtils;
046import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
047import org.junit.After;
048import org.junit.AfterClass;
049import org.junit.Before;
050import org.junit.BeforeClass;
051import org.junit.ClassRule;
052import org.junit.Ignore;
053import org.junit.Rule;
054import org.junit.Test;
055import org.junit.experimental.categories.Category;
056import org.junit.rules.TestName;
057import org.slf4j.Logger;
058import org.slf4j.LoggerFactory;
059
060import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
061import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
062
063/**
064 * Test Export Snapshot Tool
065 */
066@Ignore // HBASE-24493
067@Category({VerySlowMapReduceTests.class, LargeTests.class})
068public class TestExportSnapshot {
069
070  @ClassRule
071  public static final HBaseClassTestRule CLASS_RULE =
072      HBaseClassTestRule.forClass(TestExportSnapshot.class);
073
074  private static final Logger LOG = LoggerFactory.getLogger(TestExportSnapshot.class);
075
076  protected final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
077
078  protected final static byte[] FAMILY = Bytes.toBytes("cf");
079
080  @Rule
081  public final TestName testName = new TestName();
082
083  protected TableName tableName;
084  private String emptySnapshotName;
085  private String snapshotName;
086  private int tableNumFiles;
087  private Admin admin;
088
089  public static void setUpBaseConf(Configuration conf) {
090    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
091    conf.setInt("hbase.regionserver.msginterval", 100);
092    // If a single node has enough failures (default 3), resource manager will blacklist it.
093    // With only 2 nodes and tests injecting faults, we don't want that.
094    conf.setInt("mapreduce.job.maxtaskfailures.per.tracker", 100);
095  }
096
097  @BeforeClass
098  public static void setUpBeforeClass() throws Exception {
099    setUpBaseConf(TEST_UTIL.getConfiguration());
100    TEST_UTIL.startMiniCluster(1);
101    TEST_UTIL.startMiniMapReduceCluster();
102  }
103
104  @AfterClass
105  public static void tearDownAfterClass() throws Exception {
106    TEST_UTIL.shutdownMiniMapReduceCluster();
107    TEST_UTIL.shutdownMiniCluster();
108  }
109
110  /**
111   * Create a table and take a snapshot of the table used by the export test.
112   */
113  @Before
114  public void setUp() throws Exception {
115    this.admin = TEST_UTIL.getAdmin();
116
117    tableName = TableName.valueOf("testtb-" + testName.getMethodName());
118    snapshotName = "snaptb0-" + testName.getMethodName();
119    emptySnapshotName = "emptySnaptb0-" + testName.getMethodName();
120
121    // create Table
122    createTable(this.tableName);
123
124    // Take an empty snapshot
125    admin.snapshot(emptySnapshotName, tableName);
126
127    // Add some rows
128    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
129    tableNumFiles = admin.getRegions(tableName).size();
130
131    // take a snapshot
132    admin.snapshot(snapshotName, tableName);
133  }
134
135  protected void createTable(TableName tableName) throws Exception {
136    SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY);
137  }
138
139  protected interface RegionPredicate {
140    boolean evaluate(final RegionInfo regionInfo);
141  }
142
143  protected RegionPredicate getBypassRegionPredicate() {
144    return null;
145  }
146
147  @After
148  public void tearDown() throws Exception {
149    TEST_UTIL.deleteTable(tableName);
150    SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin());
151    SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
152  }
153
154  /**
155   * Verify if exported snapshot and copied files matches the original one.
156   */
157  @Test
158  public void testExportFileSystemState() throws Exception {
159    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
160  }
161
162  @Test
163  public void testExportFileSystemStateWithSkipTmp() throws Exception {
164    TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
165    try {
166      testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
167    } finally {
168      TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false);
169    }
170  }
171
172  @Test
173  public void testEmptyExportFileSystemState() throws Exception {
174    testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0);
175  }
176
177  @Test
178  public void testConsecutiveExports() throws Exception {
179    Path copyDir = getLocalDestinationDir(TEST_UTIL);
180    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false);
181    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true);
182    removeExportDir(copyDir);
183  }
184
185  @Test
186  public void testExportWithTargetName() throws Exception {
187    final String targetName = "testExportWithTargetName";
188    testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles);
189  }
190
191  private void testExportFileSystemState(final TableName tableName, final String snapshotName,
192      final String targetName, int filesExpected) throws Exception {
193    testExportFileSystemState(tableName, snapshotName, targetName,
194      filesExpected, getHdfsDestinationDir(), false);
195  }
196
197  protected void testExportFileSystemState(final TableName tableName,
198      final String snapshotName, final String targetName, int filesExpected,
199      Path copyDir, boolean overwrite) throws Exception {
200    testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName,
201      filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir,
202      overwrite, getBypassRegionPredicate(), true);
203  }
204
205  /**
206   * Creates destination directory, runs ExportSnapshot() tool, and runs some verifications.
207   */
208  protected static void testExportFileSystemState(final Configuration conf, final TableName tableName,
209      final String snapshotName, final String targetName, final int filesExpected,
210      final Path srcDir, Path rawTgtDir, final boolean overwrite,
211      final RegionPredicate bypassregionPredicate, boolean success) throws Exception {
212    FileSystem tgtFs = rawTgtDir.getFileSystem(conf);
213    FileSystem srcFs = srcDir.getFileSystem(conf);
214    Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory());
215    LOG.info("tgtFsUri={}, tgtDir={}, rawTgtDir={}, srcFsUri={}, srcDir={}",
216      tgtFs.getUri(), tgtDir, rawTgtDir, srcFs.getUri(), srcDir);
217    List<String> opts = new ArrayList<>();
218    opts.add("--snapshot");
219    opts.add(snapshotName);
220    opts.add("--copy-to");
221    opts.add(tgtDir.toString());
222    if (!targetName.equals(snapshotName)) {
223      opts.add("--target");
224      opts.add(targetName);
225    }
226    if (overwrite) {
227      opts.add("--overwrite");
228    }
229
230    // Export Snapshot
231    int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()]));
232    assertEquals("success " + success + ", res=" + res, success ? 0 : 1, res);
233    if (!success) {
234      final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName);
235      assertFalse(tgtDir.toString() + " " + targetDir.toString(),
236        tgtFs.exists(new Path(tgtDir, targetDir)));
237      return;
238    }
239    LOG.info("Exported snapshot");
240
241    // Verify File-System state
242    FileStatus[] rootFiles = tgtFs.listStatus(tgtDir);
243    assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length);
244    for (FileStatus fileStatus: rootFiles) {
245      String name = fileStatus.getPath().getName();
246      assertTrue(fileStatus.toString(), fileStatus.isDirectory());
247      assertTrue(name.toString(), name.equals(HConstants.SNAPSHOT_DIR_NAME) ||
248        name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY));
249    }
250    LOG.info("Verified filesystem state");
251
252    // Compare the snapshot metadata and verify the hfiles
253    final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName);
254    final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName);
255    verifySnapshotDir(srcFs, new Path(srcDir, snapshotDir), tgtFs, new Path(tgtDir, targetDir));
256    Set<String> snapshotFiles = verifySnapshot(conf, tgtFs, tgtDir, tableName,
257      targetName, bypassregionPredicate);
258    assertEquals(filesExpected, snapshotFiles.size());
259  }
260
261  /*
262   * verify if the snapshot folder on file-system 1 match the one on file-system 2
263   */
264  protected static void verifySnapshotDir(final FileSystem fs1, final Path root1,
265      final FileSystem fs2, final Path root2) throws IOException {
266    assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2));
267  }
268
269  /*
270   * Verify if the files exists
271   */
272  protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs,
273      final Path rootDir, final TableName tableName, final String snapshotName,
274      final RegionPredicate bypassregionPredicate) throws IOException {
275    final Path exportedSnapshot = new Path(rootDir,
276      new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
277    final Set<String> snapshotFiles = new HashSet<>();
278    final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
279    SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot,
280          new SnapshotReferenceUtil.SnapshotVisitor() {
281        @Override
282        public void storeFile(final RegionInfo regionInfo, final String family,
283            final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
284          if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) {
285            return;
286          }
287
288          String hfile = storeFile.getName();
289          snapshotFiles.add(hfile);
290          if (!storeFile.hasReference()) {
291            verifyNonEmptyFile(new Path(exportedArchive,
292              new Path(CommonFSUtils.getTableDir(new Path("./"), tableName),
293                  new Path(regionInfo.getEncodedName(), new Path(family, hfile)))));
294          }
295        }
296
297        private void verifyNonEmptyFile(final Path path) throws IOException {
298          assertTrue(path + " should exists", fs.exists(path));
299          assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
300        }
301      });
302
303    // Verify Snapshot description
304    SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
305    assertTrue(desc.getName().equals(snapshotName));
306    assertTrue(desc.getTable().equals(tableName.getNameAsString()));
307    return snapshotFiles;
308  }
309
310  private static Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)
311      throws IOException {
312    Set<String> files = new HashSet<>();
313    LOG.debug("List files in {} in root {} at {}", fs, root, dir);
314    int rootPrefix = root.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString().length();
315    FileStatus[] list = CommonFSUtils.listStatus(fs, dir);
316    if (list != null) {
317      for (FileStatus fstat: list) {
318        LOG.debug(Objects.toString(fstat.getPath()));
319        if (fstat.isDirectory()) {
320          files.addAll(listFiles(fs, root, fstat.getPath()));
321        } else {
322          files.add(fstat.getPath().makeQualified(fs).toString().substring(rootPrefix));
323        }
324      }
325    }
326    return files;
327  }
328
329  private Path getHdfsDestinationDir() {
330    Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
331    Path path = new Path(new Path(rootDir, "export-test"), "export-" +
332      EnvironmentEdgeManager.currentTime());
333    LOG.info("HDFS export destination path: " + path);
334    return path;
335  }
336
337  static Path getLocalDestinationDir(HBaseTestingUtil htu) {
338    Path path = htu.getDataTestDir("local-export-" + EnvironmentEdgeManager.currentTime());
339    try {
340      FileSystem fs = FileSystem.getLocal(htu.getConfiguration());
341      LOG.info("Local export destination path: " + path);
342      return path.makeQualified(fs.getUri(), fs.getWorkingDirectory());
343    } catch (IOException ioe) {
344      throw new RuntimeException(ioe);
345    }
346  }
347
348  private static void removeExportDir(final Path path) throws IOException {
349    FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
350    fs.delete(path, true);
351  }
352}