001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.jupiter.api.Assertions.assertTrue;
021
022import java.io.IOException;
023import java.util.HashSet;
024import java.util.Set;
025import org.apache.hadoop.fs.FileSystem;
026import org.apache.hadoop.fs.LocalFileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseCommonTestingUtil;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
032import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
033import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
034import org.apache.hadoop.hbase.testclassification.MapReduceTests;
035import org.apache.hadoop.hbase.testclassification.SmallTests;
036import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
037import org.apache.hadoop.hbase.util.Pair;
038import org.junit.jupiter.api.BeforeEach;
039import org.junit.jupiter.api.Tag;
040import org.junit.jupiter.api.Test;
041import org.slf4j.Logger;
042import org.slf4j.LoggerFactory;
043
044/**
045 * Test Export Snapshot Tool Tests V1 snapshots only. Used to ALSO test v2 but strange failure so
046 * separate the tests. See companion file for test of v2 snapshot.
047 * @see TestExportSnapshotV2NoCluster
048 */
049@Tag(MapReduceTests.TAG)
050@Tag(SmallTests.TAG)
051public class TestExportSnapshotV1NoCluster {
052
053  private static final Logger LOG = LoggerFactory.getLogger(TestExportSnapshotV1NoCluster.class);
054
055  private HBaseCommonTestingUtil testUtil = new HBaseCommonTestingUtil();
056  private Path testDir;
057  private FileSystem fs;
058
059  @BeforeEach
060  public void setUpBefore() throws Exception {
061    // Make sure testDir is on LocalFileSystem
062    this.fs = FileSystem.getLocal(this.testUtil.getConfiguration());
063    this.testDir = setup(fs, this.testUtil);
064    LOG.info("fs={}, fsuri={}, fswd={}, testDir={}", this.fs, this.fs.getUri(),
065      this.fs.getWorkingDirectory(), this.testDir);
066    assertTrue(fs instanceof LocalFileSystem, "FileSystem '" + fs + "' is not local");
067  }
068
069  /**
070   * Setup for test. Returns path to test data dir. Sets configuration into the passed
071   * hctu.getConfiguration.
072   */
073  static Path setup(FileSystem fs, HBaseCommonTestingUtil hctu) throws IOException {
074    Path testDir = hctu.getDataTestDir().makeQualified(fs.getUri(), fs.getWorkingDirectory());
075    hctu.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
076    hctu.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
077    hctu.getConfiguration().setInt("hbase.client.pause", 250);
078    hctu.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
079    hctu.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
080    hctu.getConfiguration().setInt("mapreduce.map.maxattempts", 10);
081    hctu.getConfiguration().set(HConstants.HBASE_DIR, testDir.toString());
082    return testDir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
083  }
084
085  /**
086   * V1 snapshot test
087   */
088  @Test
089  public void testSnapshotWithRefsExportFileSystemState() throws Exception {
090    final SnapshotMock snapshotMock =
091      new SnapshotMock(testUtil.getConfiguration(), this.fs, testDir);
092    final SnapshotMock.SnapshotBuilder builder =
093      snapshotMock.createSnapshotV1("tableWithRefsV1", "tableWithRefsV1");
094    testSnapshotWithRefsExportFileSystemState(this.fs, builder, testUtil, testDir);
095  }
096
097  /**
098   * Generates a couple of regions for the specified SnapshotMock, and then it will run the export
099   * and verification.
100   */
101  static void testSnapshotWithRefsExportFileSystemState(FileSystem fs,
102    SnapshotMock.SnapshotBuilder builder, HBaseCommonTestingUtil testUtil, Path testDir)
103    throws Exception {
104    Path[] r1Files = builder.addRegion();
105    Path[] r2Files = builder.addRegion();
106    builder.commit();
107    // remove references, only keep data files
108    Set<String> dataFiles = new HashSet<>();
109    for (Path[] files : new Path[][] { r1Files, r2Files }) {
110      for (Path file : files) {
111        if (StoreFileInfo.isReference(file.getName())) {
112          Pair<String, String> referredToRegionAndFile =
113            StoreFileInfo.getReferredToRegionAndFile(file.getName());
114          dataFiles.add(referredToRegionAndFile.getSecond());
115        } else {
116          dataFiles.add(file.getName());
117        }
118      }
119    }
120    int snapshotFilesCount = dataFiles.size();
121    String snapshotName = builder.getSnapshotDescription().getName();
122    TableName tableName = builder.getTableDescriptor().getTableName();
123    ExportSnapshotTestBase.testExportFileSystemState(testUtil.getConfiguration(), tableName,
124      snapshotName, snapshotName, snapshotFilesCount, testDir,
125      getDestinationDir(fs, testUtil, testDir), false, false, null, true, false);
126  }
127
128  static Path getDestinationDir(FileSystem fs, HBaseCommonTestingUtil hctu, Path testDir)
129    throws IOException {
130    Path path =
131      new Path(new Path(testDir, "export-test"), "export-" + EnvironmentEdgeManager.currentTime())
132        .makeQualified(fs.getUri(), fs.getWorkingDirectory());
133    LOG.info("Export destination={}, fs={}, fsurl={}, fswd={}, testDir={}", path, fs, fs.getUri(),
134      fs.getWorkingDirectory(), testDir);
135    return path;
136  }
137}