001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.snapshot;
019
020import static org.junit.jupiter.api.Assertions.assertFalse;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022
023import java.io.IOException;
024import java.util.Collection;
025import java.util.HashSet;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.fs.FileSystem;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.RegionInfo;
033import org.apache.hadoop.hbase.client.RegionInfoBuilder;
034import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
035import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
036import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
037import org.apache.hadoop.hbase.testclassification.MasterTests;
038import org.apache.hadoop.hbase.testclassification.SmallTests;
039import org.apache.hadoop.hbase.util.CommonFSUtils;
040import org.junit.jupiter.api.AfterAll;
041import org.junit.jupiter.api.BeforeAll;
042import org.junit.jupiter.api.Tag;
043import org.junit.jupiter.api.Test;
044import org.junit.jupiter.api.TestInfo;
045
046/**
047 * Test that the snapshot hfile cleaner finds hfiles referenced in a snapshot
048 */
049@Tag(MasterTests.TAG)
050@Tag(SmallTests.TAG)
051public class TestSnapshotHFileCleaner {
052
053  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
054  private static final String TABLE_NAME_STR = "testSnapshotManifest";
055  private static final String SNAPSHOT_NAME_STR = "testSnapshotManifest-snapshot";
056  private static Path rootDir;
057  private static FileSystem fs;
058  private static Configuration conf;
059
060  /**
061   * Setup the test environment
062   */
063  @BeforeAll
064  public static void setup() throws Exception {
065    conf = TEST_UTIL.getConfiguration();
066    rootDir = CommonFSUtils.getRootDir(conf);
067    fs = FileSystem.get(conf);
068  }
069
070  @AfterAll
071  public static void cleanup() throws IOException {
072    // cleanup
073    fs.delete(rootDir, true);
074  }
075
076  @Test
077  public void testFindsSnapshotFilesWhenCleaning(TestInfo testInfo) throws IOException {
078    CommonFSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
079    Path rootDir = CommonFSUtils.getRootDir(conf);
080    Path archivedHfileDir =
081      new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
082
083    FileSystem fs = FileSystem.get(conf);
084    SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner();
085    cleaner.setConf(conf);
086
087    // write an hfile to the snapshot directory
088    String snapshotName = "snapshot";
089    final TableName tableName = TableName.valueOf(testInfo.getTestMethod().get().getName());
090    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
091    RegionInfo mockRegion = RegionInfoBuilder.newBuilder(tableName).build();
092    Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName());
093    Path familyDir = new Path(regionSnapshotDir, "family");
094    // create a reference to a supposedly valid hfile
095    String hfile = "fd1e73e8a96c486090c5cec07b4894c4";
096    Path refFile = new Path(familyDir, hfile);
097
098    // make sure the reference file exists
099    fs.create(refFile);
100
101    // create the hfile in the archive
102    fs.mkdirs(archivedHfileDir);
103    fs.createNewFile(new Path(archivedHfileDir, hfile));
104
105    // make sure that the file isn't deletable
106    assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile)));
107  }
108
109  static class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
110    @Override
111    public Collection<String> filesUnderSnapshot(final FileSystem workingFs, final Path snapshotDir)
112      throws IOException {
113      Collection<String> files = new HashSet<>();
114      files.addAll(SnapshotReferenceUtil.getHFileNames(conf, workingFs, snapshotDir));
115      return files;
116    }
117  }
118
119  /**
120   * If there is a corrupted region manifest, it should throw out CorruptedSnapshotException,
121   * instead of an IOException
122   */
123  @Test
124  public void testCorruptedRegionManifest() throws IOException {
125    SnapshotTestingUtils.SnapshotMock snapshotMock =
126      new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
127    SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder =
128      snapshotMock.createSnapshotV2(SNAPSHOT_NAME_STR, TABLE_NAME_STR);
129    builder.addRegionV2();
130    builder.corruptOneRegionManifest();
131
132    long period = Long.MAX_VALUE;
133    SnapshotFileCache cache = new SnapshotFileCache(conf, period, 10000000,
134      "test-snapshot-file-cache-refresh", new SnapshotFiles());
135    try {
136      cache.getSnapshotsInProgress();
137    } finally {
138      fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, conf), true);
139    }
140  }
141
142  /**
143   * If there is a corrupted data manifest, it should throw out CorruptedSnapshotException, instead
144   * of an IOException
145   */
146  @Test
147  public void testCorruptedDataManifest() throws IOException {
148    SnapshotTestingUtils.SnapshotMock snapshotMock =
149      new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
150    SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder =
151      snapshotMock.createSnapshotV2(SNAPSHOT_NAME_STR, TABLE_NAME_STR);
152    builder.addRegionV2();
153    // consolidate to generate a data.manifest file
154    builder.consolidate();
155    builder.corruptDataManifest();
156
157    long period = Long.MAX_VALUE;
158    SnapshotFileCache cache = new SnapshotFileCache(conf, period, 10000000,
159      "test-snapshot-file-cache-refresh", new SnapshotFiles());
160    try {
161      cache.getSnapshotsInProgress();
162    } finally {
163      fs.delete(
164        SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, TEST_UTIL.getConfiguration()),
165        true);
166    }
167  }
168
169  @Test
170  public void testMissedTmpSnapshot() throws IOException {
171    SnapshotTestingUtils.SnapshotMock snapshotMock =
172      new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
173    SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder =
174      snapshotMock.createSnapshotV2(SNAPSHOT_NAME_STR, TABLE_NAME_STR);
175    builder.addRegionV2();
176    builder.missOneRegionSnapshotFile();
177    long period = Long.MAX_VALUE;
178    SnapshotFileCache cache = new SnapshotFileCache(conf, period, 10000000,
179      "test-snapshot-file-cache-refresh", new SnapshotFiles());
180    cache.getSnapshotsInProgress();
181    assertTrue(fs.exists(builder.getSnapshotsDir()));
182  }
183}