001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.snapshot;
019
020import static org.junit.Assert.assertFalse;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.Collection;
025import java.util.HashSet;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.fs.FileSystem;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.HRegionInfo;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
035import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
036import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
037import org.apache.hadoop.hbase.testclassification.MasterTests;
038import org.apache.hadoop.hbase.testclassification.SmallTests;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.apache.hadoop.hbase.util.CommonFSUtils;
041import org.junit.AfterClass;
042import org.junit.BeforeClass;
043import org.junit.ClassRule;
044import org.junit.Rule;
045import org.junit.Test;
046import org.junit.experimental.categories.Category;
047import org.junit.rules.TestName;
048
049/**
050 * Test that the snapshot hfile cleaner finds hfiles referenced in a snapshot
051 */
052@Category({ MasterTests.class, SmallTests.class })
053public class TestSnapshotHFileCleaner {
054
055  @ClassRule
056  public static final HBaseClassTestRule CLASS_RULE =
057    HBaseClassTestRule.forClass(TestSnapshotHFileCleaner.class);
058
059  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
060  private static final String TABLE_NAME_STR = "testSnapshotManifest";
061  private static final String SNAPSHOT_NAME_STR = "testSnapshotManifest-snapshot";
062  private static Path rootDir;
063  private static FileSystem fs;
064  private static Configuration conf;
065
066  @Rule
067  public TestName name = new TestName();
068
069  /**
070   * Setup the test environment
071   */
072  @BeforeClass
073  public static void setup() throws Exception {
074    conf = TEST_UTIL.getConfiguration();
075    rootDir = CommonFSUtils.getRootDir(conf);
076    fs = FileSystem.get(conf);
077  }
078
079  @AfterClass
080  public static void cleanup() throws IOException {
081    // cleanup
082    fs.delete(rootDir, true);
083  }
084
085  @Test
086  public void testFindsSnapshotFilesWhenCleaning() throws IOException {
087    CommonFSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
088    Path rootDir = CommonFSUtils.getRootDir(conf);
089    Path archivedHfileDir =
090      new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
091
092    FileSystem fs = FileSystem.get(conf);
093    SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner();
094    cleaner.setConf(conf);
095
096    // write an hfile to the snapshot directory
097    String snapshotName = "snapshot";
098    byte[] snapshot = Bytes.toBytes(snapshotName);
099    final TableName tableName = TableName.valueOf(name.getMethodName());
100    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
101    HRegionInfo mockRegion = new HRegionInfo(tableName);
102    Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName());
103    Path familyDir = new Path(regionSnapshotDir, "family");
104    // create a reference to a supposedly valid hfile
105    String hfile = "fd1e73e8a96c486090c5cec07b4894c4";
106    Path refFile = new Path(familyDir, hfile);
107
108    // make sure the reference file exists
109    fs.create(refFile);
110
111    // create the hfile in the archive
112    fs.mkdirs(archivedHfileDir);
113    fs.createNewFile(new Path(archivedHfileDir, hfile));
114
115    // make sure that the file isn't deletable
116    assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile)));
117  }
118
119  static class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
120    @Override
121    public Collection<String> filesUnderSnapshot(final FileSystem workingFs, final Path snapshotDir)
122      throws IOException {
123      Collection<String> files = new HashSet<>();
124      files.addAll(SnapshotReferenceUtil.getHFileNames(conf, workingFs, snapshotDir));
125      return files;
126    }
127  }
128
129  /**
130   * If there is a corrupted region manifest, it should throw out CorruptedSnapshotException,
131   * instead of an IOException
132   */
133  @Test
134  public void testCorruptedRegionManifest() throws IOException {
135    SnapshotTestingUtils.SnapshotMock snapshotMock =
136      new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
137    SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder =
138      snapshotMock.createSnapshotV2(SNAPSHOT_NAME_STR, TABLE_NAME_STR);
139    builder.addRegionV2();
140    builder.corruptOneRegionManifest();
141
142    long period = Long.MAX_VALUE;
143    SnapshotFileCache cache = new SnapshotFileCache(conf, period, 10000000,
144      "test-snapshot-file-cache-refresh", new SnapshotFiles());
145    try {
146      cache.getSnapshotsInProgress();
147    } finally {
148      fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, conf), true);
149    }
150  }
151
152  /**
153   * If there is a corrupted data manifest, it should throw out CorruptedSnapshotException, instead
154   * of an IOException
155   */
156  @Test
157  public void testCorruptedDataManifest() throws IOException {
158    SnapshotTestingUtils.SnapshotMock snapshotMock =
159      new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
160    SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder =
161      snapshotMock.createSnapshotV2(SNAPSHOT_NAME_STR, TABLE_NAME_STR);
162    builder.addRegionV2();
163    // consolidate to generate a data.manifest file
164    builder.consolidate();
165    builder.corruptDataManifest();
166
167    long period = Long.MAX_VALUE;
168    SnapshotFileCache cache = new SnapshotFileCache(conf, period, 10000000,
169      "test-snapshot-file-cache-refresh", new SnapshotFiles());
170    try {
171      cache.getSnapshotsInProgress();
172    } finally {
173      fs.delete(
174        SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, TEST_UTIL.getConfiguration()),
175        true);
176    }
177  }
178
179  @Test
180  public void testMissedTmpSnapshot() throws IOException {
181    SnapshotTestingUtils.SnapshotMock snapshotMock =
182      new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
183    SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder =
184      snapshotMock.createSnapshotV2(SNAPSHOT_NAME_STR, TABLE_NAME_STR);
185    builder.addRegionV2();
186    builder.missOneRegionSnapshotFile();
187    long period = Long.MAX_VALUE;
188    SnapshotFileCache cache = new SnapshotFileCache(conf, period, 10000000,
189      "test-snapshot-file-cache-refresh", new SnapshotFiles());
190    cache.getSnapshotsInProgress();
191    assertTrue(fs.exists(builder.getSnapshotsDir()));
192  }
193}