001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.apache.hadoop.util.ToolRunner.run;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.HashMap;
028import java.util.HashSet;
029import java.util.List;
030import java.util.Map;
031import java.util.Objects;
032import java.util.Optional;
033import java.util.Set;
034import java.util.stream.Collectors;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.fs.FileStatus;
037import org.apache.hadoop.fs.FileSystem;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.hbase.HBaseClassTestRule;
040import org.apache.hadoop.hbase.HBaseTestingUtil;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.client.Admin;
044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
045import org.apache.hadoop.hbase.client.Put;
046import org.apache.hadoop.hbase.client.RegionInfo;
047import org.apache.hadoop.hbase.client.Table;
048import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
049import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
050import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
051import org.apache.hadoop.hbase.testclassification.LargeTests;
052import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests;
053import org.apache.hadoop.hbase.util.Bytes;
054import org.apache.hadoop.hbase.util.CommonFSUtils;
055import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
056import org.apache.hadoop.hbase.util.Pair;
057import org.junit.After;
058import org.junit.AfterClass;
059import org.junit.Before;
060import org.junit.BeforeClass;
061import org.junit.ClassRule;
062import org.junit.Ignore;
063import org.junit.Rule;
064import org.junit.Test;
065import org.junit.experimental.categories.Category;
066import org.junit.rules.TestName;
067import org.slf4j.Logger;
068import org.slf4j.LoggerFactory;
069
070import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
071
072import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
073import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
074
075/**
076 * Test Export Snapshot Tool
077 */
078@Ignore // HBASE-24493
079@Category({ VerySlowMapReduceTests.class, LargeTests.class })
080public class TestExportSnapshot {
081
082  @ClassRule
083  public static final HBaseClassTestRule CLASS_RULE =
084    HBaseClassTestRule.forClass(TestExportSnapshot.class);
085
086  private static final Logger LOG = LoggerFactory.getLogger(TestExportSnapshot.class);
087
088  protected final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
089
090  protected final static byte[] FAMILY = Bytes.toBytes("cf");
091
092  @Rule
093  public final TestName testName = new TestName();
094
095  protected TableName tableName;
096  private String emptySnapshotName;
097  private String snapshotName;
098  private int tableNumFiles;
099  private Admin admin;
100
101  public static void setUpBaseConf(Configuration conf) {
102    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
103    conf.setInt("hbase.regionserver.msginterval", 100);
104    // If a single node has enough failures (default 3), resource manager will blacklist it.
105    // With only 2 nodes and tests injecting faults, we don't want that.
106    conf.setInt("mapreduce.job.maxtaskfailures.per.tracker", 100);
107  }
108
109  @BeforeClass
110  public static void setUpBeforeClass() throws Exception {
111    setUpBaseConf(TEST_UTIL.getConfiguration());
112    TEST_UTIL.startMiniCluster(1);
113    TEST_UTIL.startMiniMapReduceCluster();
114  }
115
116  @AfterClass
117  public static void tearDownAfterClass() throws Exception {
118    TEST_UTIL.shutdownMiniMapReduceCluster();
119    TEST_UTIL.shutdownMiniCluster();
120  }
121
122  /**
123   * Create a table and take a snapshot of the table used by the export test.
124   */
125  @Before
126  public void setUp() throws Exception {
127    this.admin = TEST_UTIL.getAdmin();
128
129    tableName = TableName.valueOf("testtb-" + testName.getMethodName());
130    snapshotName = "snaptb0-" + testName.getMethodName();
131    emptySnapshotName = "emptySnaptb0-" + testName.getMethodName();
132
133    // create Table
134    createTable(this.tableName);
135
136    // Take an empty snapshot
137    admin.snapshot(emptySnapshotName, tableName);
138
139    // Add some rows
140    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
141    tableNumFiles = admin.getRegions(tableName).size();
142
143    // take a snapshot
144    admin.snapshot(snapshotName, tableName);
145  }
146
147  protected void createTable(TableName tableName) throws Exception {
148    SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY);
149  }
150
151  protected interface RegionPredicate {
152    boolean evaluate(final RegionInfo regionInfo);
153  }
154
155  protected RegionPredicate getBypassRegionPredicate() {
156    return null;
157  }
158
159  @After
160  public void tearDown() throws Exception {
161    TEST_UTIL.deleteTable(tableName);
162    SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin());
163    SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
164  }
165
166  /**
167   * Verify if exported snapshot and copied files matches the original one.
168   */
169  @Test
170  public void testExportFileSystemState() throws Exception {
171    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
172  }
173
174  @Test
175  public void testExportFileSystemStateWithMergeRegion() throws Exception {
176    // disable compaction
177    admin.compactionSwitch(false,
178      admin.getRegionServers().stream().map(a -> a.getServerName()).collect(Collectors.toList()));
179    // create Table
180    TableName tableName0 = TableName.valueOf("testtb-" + testName.getMethodName() + "-1");
181    String snapshotName0 = "snaptb0-" + testName.getMethodName() + "-1";
182    admin.createTable(
183      TableDescriptorBuilder.newBuilder(tableName0)
184        .setColumnFamilies(
185          Lists.newArrayList(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build()))
186        .build(),
187      new byte[][] { Bytes.toBytes("2") });
188    // put some data
189    try (Table table = admin.getConnection().getTable(tableName0)) {
190      table.put(new Put(Bytes.toBytes("1")).addColumn(FAMILY, null, Bytes.toBytes("1")));
191      table.put(new Put(Bytes.toBytes("2")).addColumn(FAMILY, null, Bytes.toBytes("2")));
192    }
193    List<RegionInfo> regions = admin.getRegions(tableName0);
194    assertEquals(2, regions.size());
195    tableNumFiles = regions.size();
196    // merge region
197    admin.mergeRegionsAsync(new byte[][] { regions.get(0).getEncodedNameAsBytes(),
198      regions.get(1).getEncodedNameAsBytes() }, true).get();
199    // take a snapshot
200    admin.snapshot(snapshotName0, tableName0);
201    // export snapshot and verify
202    testExportFileSystemState(tableName0, snapshotName0, snapshotName0, tableNumFiles);
203    // delete table
204    TEST_UTIL.deleteTable(tableName0);
205  }
206
207  @Test
208  public void testExportFileSystemStateWithSkipTmp() throws Exception {
209    TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
210    try {
211      testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
212    } finally {
213      TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false);
214    }
215  }
216
217  @Test
218  public void testEmptyExportFileSystemState() throws Exception {
219    testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0);
220  }
221
222  @Test
223  public void testConsecutiveExports() throws Exception {
224    Path copyDir = getLocalDestinationDir(TEST_UTIL);
225    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false);
226    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true);
227    removeExportDir(copyDir);
228  }
229
230  @Test
231  public void testExportWithTargetName() throws Exception {
232    final String targetName = "testExportWithTargetName";
233    testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles);
234  }
235
236  @Test
237  public void testExportWithResetTtl() throws Exception {
238    String name = "testExportWithResetTtl";
239    TableName tableName = TableName.valueOf(name);
240    String snapshotName = "snaptb-" + name;
241    Long ttl = 100000L;
242
243    try {
244      // create Table
245      createTable(tableName);
246      SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
247      int tableNumFiles = admin.getRegions(tableName).size();
248      // take a snapshot with TTL
249      Map<String, Object> props = new HashMap<>();
250      props.put("TTL", ttl);
251      admin.snapshot(snapshotName, tableName, props);
252      Optional<Long> ttlOpt =
253        admin.listSnapshots().stream().filter(s -> s.getName().equals(snapshotName))
254          .map(org.apache.hadoop.hbase.client.SnapshotDescription::getTtl).findAny();
255      assertTrue(ttlOpt.isPresent());
256      assertEquals(ttl, ttlOpt.get());
257
258      testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles,
259        getHdfsDestinationDir(), false, true);
260    } catch (Exception e) {
261      throw e;
262    } finally {
263      TEST_UTIL.deleteTable(tableName);
264    }
265  }
266
267  private void testExportFileSystemState(final TableName tableName, final String snapshotName,
268    final String targetName, int filesExpected) throws Exception {
269    testExportFileSystemState(tableName, snapshotName, targetName, filesExpected,
270      getHdfsDestinationDir(), false);
271  }
272
273  protected void testExportFileSystemState(final TableName tableName, final String snapshotName,
274    final String targetName, int filesExpected, Path copyDir, boolean overwrite) throws Exception {
275    testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir,
276      overwrite, false);
277  }
278
279  protected void testExportFileSystemState(final TableName tableName, final String snapshotName,
280    final String targetName, int filesExpected, Path copyDir, boolean overwrite, boolean resetTtl)
281    throws Exception {
282    testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName,
283      filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite, resetTtl,
284      getBypassRegionPredicate(), true);
285  }
286
287  /**
288   * Creates destination directory, runs ExportSnapshot() tool, and runs some verifications.
289   */
290  protected static void testExportFileSystemState(final Configuration conf,
291    final TableName tableName, final String snapshotName, final String targetName,
292    final int filesExpected, final Path srcDir, Path rawTgtDir, final boolean overwrite,
293    final boolean resetTtl, final RegionPredicate bypassregionPredicate, boolean success)
294    throws Exception {
295    FileSystem tgtFs = rawTgtDir.getFileSystem(conf);
296    FileSystem srcFs = srcDir.getFileSystem(conf);
297    Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory());
298    LOG.info("tgtFsUri={}, tgtDir={}, rawTgtDir={}, srcFsUri={}, srcDir={}", tgtFs.getUri(), tgtDir,
299      rawTgtDir, srcFs.getUri(), srcDir);
300    List<String> opts = new ArrayList<>();
301    opts.add("--snapshot");
302    opts.add(snapshotName);
303    opts.add("--copy-to");
304    opts.add(tgtDir.toString());
305    if (!targetName.equals(snapshotName)) {
306      opts.add("--target");
307      opts.add(targetName);
308    }
309    if (overwrite) {
310      opts.add("--overwrite");
311    }
312    if (resetTtl) {
313      opts.add("--reset-ttl");
314    }
315
316    // Export Snapshot
317    int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()]));
318    assertEquals("success " + success + ", res=" + res, success ? 0 : 1, res);
319    if (!success) {
320      final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName);
321      assertFalse(tgtDir.toString() + " " + targetDir.toString(),
322        tgtFs.exists(new Path(tgtDir, targetDir)));
323      return;
324    }
325    LOG.info("Exported snapshot");
326
327    // Verify File-System state
328    FileStatus[] rootFiles = tgtFs.listStatus(tgtDir);
329    assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length);
330    for (FileStatus fileStatus : rootFiles) {
331      String name = fileStatus.getPath().getName();
332      assertTrue(fileStatus.toString(), fileStatus.isDirectory());
333      assertTrue(name.toString(), name.equals(HConstants.SNAPSHOT_DIR_NAME)
334        || name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY));
335    }
336    LOG.info("Verified filesystem state");
337
338    // Compare the snapshot metadata and verify the hfiles
339    final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName);
340    final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName);
341    verifySnapshotDir(srcFs, new Path(srcDir, snapshotDir), tgtFs, new Path(tgtDir, targetDir));
342    Set<String> snapshotFiles =
343      verifySnapshot(conf, tgtFs, tgtDir, tableName, targetName, resetTtl, bypassregionPredicate);
344    assertEquals(filesExpected, snapshotFiles.size());
345  }
346
347  /*
348   * verify if the snapshot folder on file-system 1 match the one on file-system 2
349   */
350  protected static void verifySnapshotDir(final FileSystem fs1, final Path root1,
351    final FileSystem fs2, final Path root2) throws IOException {
352    assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2));
353  }
354
355  /*
356   * Verify if the files exists
357   */
358  protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs,
359    final Path rootDir, final TableName tableName, final String snapshotName,
360    final boolean resetTtl, final RegionPredicate bypassregionPredicate) throws IOException {
361    final Path exportedSnapshot =
362      new Path(rootDir, new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
363    final Set<String> snapshotFiles = new HashSet<>();
364    final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
365    SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot,
366      new SnapshotReferenceUtil.SnapshotVisitor() {
367        @Override
368        public void storeFile(final RegionInfo regionInfo, final String family,
369          final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
370          if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) {
371            return;
372          }
373
374          if (!storeFile.hasReference() && !StoreFileInfo.isReference(storeFile.getName())) {
375            String hfile = storeFile.getName();
376            snapshotFiles.add(hfile);
377            verifyNonEmptyFile(new Path(exportedArchive,
378              new Path(CommonFSUtils.getTableDir(new Path("./"), tableName),
379                new Path(regionInfo.getEncodedName(), new Path(family, hfile)))));
380          } else {
381            Pair<String, String> referredToRegionAndFile =
382              StoreFileInfo.getReferredToRegionAndFile(storeFile.getName());
383            String region = referredToRegionAndFile.getFirst();
384            String hfile = referredToRegionAndFile.getSecond();
385            snapshotFiles.add(hfile);
386            verifyNonEmptyFile(new Path(exportedArchive,
387              new Path(CommonFSUtils.getTableDir(new Path("./"), tableName),
388                new Path(region, new Path(family, hfile)))));
389          }
390        }
391
392        private void verifyNonEmptyFile(final Path path) throws IOException {
393          assertTrue(path + " should exists", fs.exists(path));
394          assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
395        }
396      });
397
398    // Verify Snapshot description
399    SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
400    assertTrue(desc.getName().equals(snapshotName));
401    assertTrue(desc.getTable().equals(tableName.getNameAsString()));
402    if (resetTtl) {
403      assertEquals(HConstants.DEFAULT_SNAPSHOT_TTL, desc.getTtl());
404    }
405    return snapshotFiles;
406  }
407
408  private static Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)
409    throws IOException {
410    Set<String> files = new HashSet<>();
411    LOG.debug("List files in {} in root {} at {}", fs, root, dir);
412    int rootPrefix = root.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString().length();
413    FileStatus[] list = CommonFSUtils.listStatus(fs, dir);
414    if (list != null) {
415      for (FileStatus fstat : list) {
416        LOG.debug(Objects.toString(fstat.getPath()));
417        if (fstat.isDirectory()) {
418          files.addAll(listFiles(fs, root, fstat.getPath()));
419        } else {
420          files.add(fstat.getPath().makeQualified(fs).toString().substring(rootPrefix));
421        }
422      }
423    }
424    return files;
425  }
426
427  private Path getHdfsDestinationDir() {
428    Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
429    Path path =
430      new Path(new Path(rootDir, "export-test"), "export-" + EnvironmentEdgeManager.currentTime());
431    LOG.info("HDFS export destination path: " + path);
432    return path;
433  }
434
435  static Path getLocalDestinationDir(HBaseTestingUtil htu) {
436    Path path = htu.getDataTestDir("local-export-" + EnvironmentEdgeManager.currentTime());
437    try {
438      FileSystem fs = FileSystem.getLocal(htu.getConfiguration());
439      LOG.info("Local export destination path: " + path);
440      return path.makeQualified(fs.getUri(), fs.getWorkingDirectory());
441    } catch (IOException ioe) {
442      throw new RuntimeException(ioe);
443    }
444  }
445
446  private static void removeExportDir(final Path path) throws IOException {
447    FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
448    fs.delete(path, true);
449  }
450}