001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.Assert.assertTrue;
021
022import java.io.IOException;
023import java.util.HashSet;
024import java.util.Set;
025import org.apache.hadoop.fs.FileSystem;
026import org.apache.hadoop.fs.LocalFileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseCommonTestingUtil;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
033import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
034import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
035import org.apache.hadoop.hbase.testclassification.MapReduceTests;
036import org.apache.hadoop.hbase.testclassification.MediumTests;
037import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
038import org.apache.hadoop.hbase.util.Pair;
039import org.junit.Before;
040import org.junit.ClassRule;
041import org.junit.Test;
042import org.junit.experimental.categories.Category;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046/**
047 * Test Export Snapshot Tool Tests V1 snapshots only. Used to ALSO test v2 but strange failure so
048 * separate the tests. See companion file for test of v2 snapshot.
049 * @see TestExportSnapshotV2NoCluster
050 */
051@Category({ MapReduceTests.class, MediumTests.class })
052public class TestExportSnapshotV1NoCluster {
053  @ClassRule
054  public static final HBaseClassTestRule CLASS_RULE =
055    HBaseClassTestRule.forClass(TestExportSnapshotV1NoCluster.class);
056  private static final Logger LOG = LoggerFactory.getLogger(TestExportSnapshotV1NoCluster.class);
057
058  private HBaseCommonTestingUtil testUtil = new HBaseCommonTestingUtil();
059  private Path testDir;
060  private FileSystem fs;
061
062  @Before
063  public void setUpBefore() throws Exception {
064    // Make sure testDir is on LocalFileSystem
065    this.fs = FileSystem.getLocal(this.testUtil.getConfiguration());
066    this.testDir = setup(fs, this.testUtil);
067    LOG.info("fs={}, fsuri={}, fswd={}, testDir={}", this.fs, this.fs.getUri(),
068      this.fs.getWorkingDirectory(), this.testDir);
069    assertTrue("FileSystem '" + fs + "' is not local", fs instanceof LocalFileSystem);
070  }
071
072  /**
073   * Setup for test. Returns path to test data dir. Sets configuration into the passed
074   * hctu.getConfiguration.
075   */
076  static Path setup(FileSystem fs, HBaseCommonTestingUtil hctu) throws IOException {
077    Path testDir = hctu.getDataTestDir().makeQualified(fs.getUri(), fs.getWorkingDirectory());
078    hctu.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
079    hctu.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
080    hctu.getConfiguration().setInt("hbase.client.pause", 250);
081    hctu.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
082    hctu.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
083    hctu.getConfiguration().setInt("mapreduce.map.maxattempts", 10);
084    hctu.getConfiguration().set(HConstants.HBASE_DIR, testDir.toString());
085    return testDir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
086  }
087
088  /**
089   * V1 snapshot test
090   */
091  @Test
092  public void testSnapshotWithRefsExportFileSystemState() throws Exception {
093    final SnapshotMock snapshotMock =
094      new SnapshotMock(testUtil.getConfiguration(), this.fs, testDir);
095    final SnapshotMock.SnapshotBuilder builder =
096      snapshotMock.createSnapshotV1("tableWithRefsV1", "tableWithRefsV1");
097    testSnapshotWithRefsExportFileSystemState(this.fs, builder, testUtil, testDir);
098  }
099
100  /**
101   * Generates a couple of regions for the specified SnapshotMock, and then it will run the export
102   * and verification.
103   */
104  static void testSnapshotWithRefsExportFileSystemState(FileSystem fs,
105    SnapshotMock.SnapshotBuilder builder, HBaseCommonTestingUtil testUtil, Path testDir)
106    throws Exception {
107    Path[] r1Files = builder.addRegion();
108    Path[] r2Files = builder.addRegion();
109    builder.commit();
110    // remove references, only keep data files
111    Set<String> dataFiles = new HashSet<>();
112    for (Path[] files : new Path[][] { r1Files, r2Files }) {
113      for (Path file : files) {
114        if (StoreFileInfo.isReference(file.getName())) {
115          Pair<String, String> referredToRegionAndFile =
116            StoreFileInfo.getReferredToRegionAndFile(file.getName());
117          dataFiles.add(referredToRegionAndFile.getSecond());
118        } else {
119          dataFiles.add(file.getName());
120        }
121      }
122    }
123    int snapshotFilesCount = dataFiles.size();
124    String snapshotName = builder.getSnapshotDescription().getName();
125    TableName tableName = builder.getTableDescriptor().getTableName();
126    TestExportSnapshot.testExportFileSystemState(testUtil.getConfiguration(), tableName,
127      snapshotName, snapshotName, snapshotFilesCount, testDir,
128      getDestinationDir(fs, testUtil, testDir), false, false, null, true);
129  }
130
131  static Path getDestinationDir(FileSystem fs, HBaseCommonTestingUtil hctu, Path testDir)
132    throws IOException {
133    Path path =
134      new Path(new Path(testDir, "export-test"), "export-" + EnvironmentEdgeManager.currentTime())
135        .makeQualified(fs.getUri(), fs.getWorkingDirectory());
136    LOG.info("Export destination={}, fs={}, fsurl={}, fswd={}, testDir={}", path, fs, fs.getUri(),
137      fs.getWorkingDirectory(), testDir);
138    return path;
139  }
140}