Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.apache.hadoop.util.ToolRunner.run;
021import static org.junit.jupiter.api.Assertions.assertEquals;
022import static org.junit.jupiter.api.Assertions.assertFalse;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import java.lang.reflect.Method;
027import java.util.ArrayList;
028import java.util.HashSet;
029import java.util.List;
030import java.util.Objects;
031import java.util.Set;
032import java.util.stream.Stream;
033import org.apache.hadoop.conf.Configuration;
034import org.apache.hadoop.fs.FileStatus;
035import org.apache.hadoop.fs.FileSystem;
036import org.apache.hadoop.fs.Path;
037import org.apache.hadoop.hbase.HBaseTestingUtil;
038import org.apache.hadoop.hbase.HConstants;
039import org.apache.hadoop.hbase.TableName;
040import org.apache.hadoop.hbase.client.Admin;
041import org.apache.hadoop.hbase.client.RegionInfo;
042import org.apache.hadoop.hbase.mob.MobUtils;
043import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.hbase.util.CommonFSUtils;
046import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
047import org.apache.hadoop.hbase.util.Pair;
048import org.junit.jupiter.api.AfterAll;
049import org.junit.jupiter.api.AfterEach;
050import org.junit.jupiter.api.BeforeEach;
051import org.junit.jupiter.api.TestInfo;
052import org.junit.jupiter.params.provider.Arguments;
053import org.slf4j.Logger;
054import org.slf4j.LoggerFactory;
055
056import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
057import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
058
059public class ExportSnapshotTestBase {
060
061  private static final Logger LOG = LoggerFactory.getLogger(ExportSnapshotTestBase.class);
062
063  protected final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
064
065  protected final static byte[] FAMILY = Bytes.toBytes("cf");
066
067  protected String methodName;
068
069  protected TableName tableName;
070
071  protected String emptySnapshotName;
072
073  protected String snapshotName;
074
075  protected int tableNumFiles;
076
077  protected Admin admin;
078
079  protected boolean mob;
080
081  public static Stream<Arguments> parameters() {
082    return Stream.of(Arguments.of(false), Arguments.of(true));
083  }
084
085  protected ExportSnapshotTestBase(boolean mob) {
086    this.mob = mob;
087  }
088
089  @AfterAll
090  public static void tearDownAfterClass() throws Exception {
091    TEST_UTIL.shutdownMiniMapReduceCluster();
092    TEST_UTIL.shutdownMiniCluster();
093  }
094
095  /**
096   * Create a table and take a snapshot of the table used by the export test.
097   */
098  @BeforeEach
099  public void setUp(TestInfo testInfo) throws Exception {
100    this.admin = TEST_UTIL.getAdmin();
101
102    methodName = testInfo.getTestMethod().map(Method::getName).orElse(getClass().getSimpleName());
103
104    String suffix = mob ? methodName + "-mob" : methodName;
105    tableName = TableName.valueOf("testtb-" + suffix);
106    snapshotName = "snaptb0-" + suffix;
107    emptySnapshotName = "emptySnaptb0-" + suffix;
108
109    // create Table
110    createTable(this.tableName);
111
112    // Take an empty snapshot
113    admin.snapshot(emptySnapshotName, tableName);
114
115    // Add some rows
116    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
117    tableNumFiles = admin.getRegions(tableName).size();
118
119    // take a snapshot
120    admin.snapshot(snapshotName, tableName);
121  }
122
123  protected final void createTable(TableName tableName) throws Exception {
124    if (mob) {
125      MobSnapshotTestingUtils.createPreSplitMobTable(TEST_UTIL, tableName, 2, FAMILY);
126    } else {
127      SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY);
128    }
129  }
130
131  protected interface RegionPredicate {
132    boolean evaluate(final RegionInfo regionInfo);
133  }
134
135  protected final RegionPredicate getBypassRegionPredicate() {
136    if (mob) {
137      return MobUtils::isMobRegionInfo;
138    } else {
139      return null;
140    }
141  }
142
143  @AfterEach
144  public void tearDown() throws Exception {
145    for (TableName tn : TEST_UTIL.getAdmin().listTableNames()) {
146      TEST_UTIL.deleteTable(tn);
147    }
148    SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin());
149    SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
150  }
151
152  protected final void testExportFileSystemState(final TableName tableName,
153    final String snapshotName, final String targetName, int filesExpected) throws Exception {
154    testExportFileSystemState(tableName, snapshotName, targetName, filesExpected,
155      getHdfsDestinationDir(), false);
156  }
157
158  protected final void testExportFileSystemState(final TableName tableName,
159    final String snapshotName, final String targetName, int filesExpected, Path copyDir,
160    boolean overwrite) throws Exception {
161    testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir,
162      overwrite, false);
163  }
164
165  protected final void testExportFileSystemState(final TableName tableName,
166    final String snapshotName, final String targetName, int filesExpected, Path copyDir,
167    boolean overwrite, boolean resetTtl) throws Exception {
168    testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName,
169      filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite, resetTtl,
170      getBypassRegionPredicate(), true, false);
171  }
172
173  /**
174   * Creates destination directory, runs ExportSnapshot() tool, and runs some verifications.
175   */
176  protected static void testExportFileSystemState(final Configuration conf,
177    final TableName tableName, final String snapshotName, final String targetName,
178    final int filesExpected, final Path srcDir, Path rawTgtDir, final boolean overwrite,
179    final boolean resetTtl, final RegionPredicate bypassregionPredicate, final boolean success,
180    final boolean checksumVerify) throws Exception {
181    FileSystem tgtFs = rawTgtDir.getFileSystem(conf);
182    FileSystem srcFs = srcDir.getFileSystem(conf);
183    Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory());
184
185    // Export Snapshot
186    int res = runExportSnapshot(conf, snapshotName, targetName, srcDir, rawTgtDir, overwrite,
187      resetTtl, checksumVerify, true, true);
188    assertEquals(success ? 0 : 1, res, "success " + success + ", res=" + res);
189    if (!success) {
190      final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName);
191      assertFalse(tgtFs.exists(new Path(tgtDir, targetDir)),
192        tgtDir.toString() + " " + targetDir.toString());
193      return;
194    }
195    LOG.info("Exported snapshot");
196
197    // Verify File-System state
198    FileStatus[] rootFiles = tgtFs.listStatus(tgtDir);
199    assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length);
200    for (FileStatus fileStatus : rootFiles) {
201      String name = fileStatus.getPath().getName();
202      assertTrue(fileStatus.isDirectory(), fileStatus.toString());
203      assertTrue(name.equals(HConstants.SNAPSHOT_DIR_NAME)
204        || name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY), name.toString());
205    }
206    LOG.info("Verified filesystem state");
207
208    // Compare the snapshot metadata and verify the hfiles
209    final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName);
210    final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName);
211    verifySnapshotDir(srcFs, new Path(srcDir, snapshotDir), tgtFs, new Path(tgtDir, targetDir));
212    Set<String> snapshotFiles =
213      verifySnapshot(conf, tgtFs, tgtDir, tableName, targetName, resetTtl, bypassregionPredicate);
214    assertEquals(filesExpected, snapshotFiles.size());
215  }
216
217  /*
218   * verify if the snapshot folder on file-system 1 match the one on file-system 2
219   */
220  protected static void verifySnapshotDir(final FileSystem fs1, final Path root1,
221    final FileSystem fs2, final Path root2) throws IOException {
222    assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2));
223  }
224
225  /*
226   * Verify if the files exists
227   */
228  protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs,
229    final Path rootDir, final TableName tableName, final String snapshotName,
230    final boolean resetTtl, final RegionPredicate bypassregionPredicate) throws IOException {
231    final Path exportedSnapshot =
232      new Path(rootDir, new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
233    final Set<String> snapshotFiles = new HashSet<>();
234    final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
235    SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot,
236      new SnapshotReferenceUtil.SnapshotVisitor() {
237        @Override
238        public void storeFile(final RegionInfo regionInfo, final String family,
239          final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
240          if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) {
241            return;
242          }
243
244          if (!storeFile.hasReference() && !StoreFileInfo.isReference(storeFile.getName())) {
245            String hfile = storeFile.getName();
246            snapshotFiles.add(hfile);
247            verifyNonEmptyFile(new Path(exportedArchive,
248              new Path(CommonFSUtils.getTableDir(new Path("./"), tableName),
249                new Path(regionInfo.getEncodedName(), new Path(family, hfile)))));
250          } else {
251            Pair<String, String> referredToRegionAndFile =
252              StoreFileInfo.getReferredToRegionAndFile(storeFile.getName());
253            String region = referredToRegionAndFile.getFirst();
254            String hfile = referredToRegionAndFile.getSecond();
255            snapshotFiles.add(hfile);
256            verifyNonEmptyFile(new Path(exportedArchive,
257              new Path(CommonFSUtils.getTableDir(new Path("./"), tableName),
258                new Path(region, new Path(family, hfile)))));
259          }
260        }
261
262        private void verifyNonEmptyFile(final Path path) throws IOException {
263          assertTrue(fs.exists(path), path + " should exists");
264          assertTrue(fs.getFileStatus(path).getLen() > 0, path + " should not be empty");
265        }
266      });
267
268    // Verify Snapshot description
269    SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
270    assertTrue(desc.getName().equals(snapshotName));
271    assertTrue(desc.getTable().equals(tableName.getNameAsString()));
272    if (resetTtl) {
273      assertEquals(HConstants.DEFAULT_SNAPSHOT_TTL, desc.getTtl());
274    }
275    return snapshotFiles;
276  }
277
278  private static Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)
279    throws IOException {
280    Set<String> files = new HashSet<>();
281    LOG.debug("List files in {} in root {} at {}", fs, root, dir);
282    int rootPrefix = root.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString().length();
283    FileStatus[] list = CommonFSUtils.listStatus(fs, dir);
284    if (list != null) {
285      for (FileStatus fstat : list) {
286        LOG.debug(Objects.toString(fstat.getPath()));
287        if (fstat.isDirectory()) {
288          files.addAll(listFiles(fs, root, fstat.getPath()));
289        } else {
290          files.add(fstat.getPath().makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString()
291            .substring(rootPrefix));
292        }
293      }
294    }
295    return files;
296  }
297
298  protected final Path getHdfsDestinationDir() {
299    Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
300    Path path =
301      new Path(new Path(rootDir, "export-test"), "export-" + EnvironmentEdgeManager.currentTime());
302    LOG.info("HDFS export destination path: " + path);
303    return path;
304  }
305
306  protected static Path getLocalDestinationDir(HBaseTestingUtil htu) {
307    Path path = htu.getDataTestDir("local-export-" + EnvironmentEdgeManager.currentTime());
308    try {
309      FileSystem fs = FileSystem.getLocal(htu.getConfiguration());
310      LOG.info("Local export destination path: " + path);
311      return path.makeQualified(fs.getUri(), fs.getWorkingDirectory());
312    } catch (IOException ioe) {
313      throw new RuntimeException(ioe);
314    }
315  }
316
317  protected static void removeExportDir(final Path path) throws IOException {
318    FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
319    fs.delete(path, true);
320  }
321
322  protected static int runExportSnapshot(final Configuration conf, final String sourceSnapshotName,
323    final String targetSnapshotName, final Path srcDir, Path rawTgtDir, final boolean overwrite,
324    final boolean resetTtl, final boolean checksumVerify, final boolean noSourceVerify,
325    final boolean noTargetVerify) throws Exception {
326    FileSystem tgtFs = rawTgtDir.getFileSystem(conf);
327    FileSystem srcFs = srcDir.getFileSystem(conf);
328    Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory());
329    LOG.info("tgtFsUri={}, tgtDir={}, rawTgtDir={}, srcFsUri={}, srcDir={}", tgtFs.getUri(), tgtDir,
330      rawTgtDir, srcFs.getUri(), srcDir);
331    List<String> opts = new ArrayList<>();
332    opts.add("--snapshot");
333    opts.add(sourceSnapshotName);
334    opts.add("--copy-to");
335    opts.add(tgtDir.toString());
336    if (!targetSnapshotName.equals(sourceSnapshotName)) {
337      opts.add("--target");
338      opts.add(targetSnapshotName);
339    }
340    if (overwrite) {
341      opts.add("--overwrite");
342    }
343    if (resetTtl) {
344      opts.add("--reset-ttl");
345    }
346    if (!checksumVerify) {
347      opts.add("--no-checksum-verify");
348    }
349    if (!noSourceVerify) {
350      opts.add("--no-source-verify");
351    }
352    if (!noTargetVerify) {
353      opts.add("--no-target-verify");
354    }
355
356    // Export Snapshot
357    return run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()]));
358  }
359}