001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.snapshot;
019
020import static org.junit.Assert.assertFalse;
021
022import java.io.IOException;
023import java.util.Collection;
024import java.util.HashSet;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtility;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.HRegionInfo;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
034import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
035import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
036import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
037import org.apache.hadoop.hbase.testclassification.MasterTests;
038import org.apache.hadoop.hbase.testclassification.SmallTests;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.apache.hadoop.hbase.util.FSUtils;
041import org.junit.AfterClass;
042import org.junit.BeforeClass;
043import org.junit.ClassRule;
044import org.junit.Rule;
045import org.junit.Test;
046import org.junit.experimental.categories.Category;
047import org.junit.rules.TestName;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051/**
052 * Test that the snapshot hfile cleaner finds hfiles referenced in a snapshot
053 */
054@Category({MasterTests.class, SmallTests.class})
055public class TestSnapshotHFileCleaner {
056
057  @ClassRule
058  public static final HBaseClassTestRule CLASS_RULE =
059      HBaseClassTestRule.forClass(TestSnapshotHFileCleaner.class);
060
061  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFileCache.class);
062  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
063  private static final String TABLE_NAME_STR = "testSnapshotManifest";
064  private static final String SNAPSHOT_NAME_STR = "testSnapshotManifest-snapshot";
065  private static Path rootDir;
066  private static FileSystem fs;
067
068  @Rule
069  public TestName name = new TestName();
070
071  /**
072   * Setup the test environment
073   */
074  @BeforeClass
075  public static void setup() throws Exception {
076    Configuration conf = TEST_UTIL.getConfiguration();
077    rootDir = FSUtils.getRootDir(conf);
078    fs = FileSystem.get(conf);
079  }
080
081
082  @AfterClass
083  public static void cleanup() throws IOException {
084    // cleanup
085    fs.delete(rootDir, true);
086  }
087
088  @Test
089  public void testFindsSnapshotFilesWhenCleaning() throws IOException {
090    Configuration conf = TEST_UTIL.getConfiguration();
091    FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
092    Path rootDir = FSUtils.getRootDir(conf);
093    Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
094
095    FileSystem fs = FileSystem.get(conf);
096    SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner();
097    cleaner.setConf(conf);
098
099    // write an hfile to the snapshot directory
100    String snapshotName = "snapshot";
101    byte[] snapshot = Bytes.toBytes(snapshotName);
102    final TableName tableName = TableName.valueOf(name.getMethodName());
103    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
104    HRegionInfo mockRegion = new HRegionInfo(tableName);
105    Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName());
106    Path familyDir = new Path(regionSnapshotDir, "family");
107    // create a reference to a supposedly valid hfile
108    String hfile = "fd1e73e8a96c486090c5cec07b4894c4";
109    Path refFile = new Path(familyDir, hfile);
110
111    // make sure the reference file exists
112    fs.create(refFile);
113
114    // create the hfile in the archive
115    fs.mkdirs(archivedHfileDir);
116    fs.createNewFile(new Path(archivedHfileDir, hfile));
117
118    // make sure that the file isn't deletable
119    assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile)));
120  }
121
122  static class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
123    @Override
124    public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
125      Collection<String> files =  new HashSet<>();
126      files.addAll(SnapshotReferenceUtil.getHFileNames(TEST_UTIL.getConfiguration(), fs, snapshotDir));
127      return files;
128    }
129  }
130
131  /**
132   * If there is a corrupted region manifest, it should throw out CorruptedSnapshotException,
133   * instead of an IOException
134   */
135  @Test
136  public void testCorruptedRegionManifest() throws IOException {
137    SnapshotTestingUtils.SnapshotMock
138        snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
139    SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(
140        SNAPSHOT_NAME_STR, TABLE_NAME_STR);
141    builder.addRegionV2();
142    builder.corruptOneRegionManifest();
143
144    long period = Long.MAX_VALUE;
145    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
146        "test-snapshot-file-cache-refresh", new SnapshotFiles());
147    try {
148      cache.getSnapshotsInProgress();
149    } catch (CorruptedSnapshotException cse) {
150      LOG.info("Expected exception " + cse);
151    } finally {
152      fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true);
153    }
154  }
155
156  /**
157   * If there is a corrupted data manifest, it should throw out CorruptedSnapshotException,
158   * instead of an IOException
159   */
160  @Test
161  public void testCorruptedDataManifest() throws IOException {
162    SnapshotTestingUtils.SnapshotMock
163        snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
164    SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(
165        SNAPSHOT_NAME_STR, TABLE_NAME_STR);
166    builder.addRegionV2();
167    // consolidate to generate a data.manifest file
168    builder.consolidate();
169    builder.corruptDataManifest();
170
171    long period = Long.MAX_VALUE;
172    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
173        "test-snapshot-file-cache-refresh", new SnapshotFiles());
174    try {
175      cache.getSnapshotsInProgress();
176    } catch (CorruptedSnapshotException cse) {
177      LOG.info("Expected exception " + cse);
178    } finally {
179      fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true);
180    }
181  }
182
183  /**
184  * HBASE-16464
185  */
186  @Test
187  public void testMissedTmpSnapshot() throws IOException {
188    SnapshotTestingUtils.SnapshotMock
189        snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
190    SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(
191        SNAPSHOT_NAME_STR, TABLE_NAME_STR);
192    builder.addRegionV2();
193    builder.missOneRegionSnapshotFile();
194
195      long period = Long.MAX_VALUE;
196    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
197        "test-snapshot-file-cache-refresh", new SnapshotFiles());
198    cache.getSnapshotsInProgress();
199    assertFalse(fs.exists(builder.getSnapshotsDir()));
200  }
201}