001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.snapshot; 019 020import static org.junit.Assert.assertFalse; 021 022import java.io.IOException; 023import java.util.Collection; 024import java.util.HashSet; 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.fs.FileSystem; 027import org.apache.hadoop.fs.Path; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseTestingUtility; 030import org.apache.hadoop.hbase.HConstants; 031import org.apache.hadoop.hbase.HRegionInfo; 032import org.apache.hadoop.hbase.TableName; 033import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException; 034import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; 035import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; 036import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; 037import org.apache.hadoop.hbase.testclassification.MasterTests; 038import org.apache.hadoop.hbase.testclassification.SmallTests; 039import org.apache.hadoop.hbase.util.Bytes; 040import org.apache.hadoop.hbase.util.FSUtils; 041import org.junit.AfterClass; 042import org.junit.BeforeClass; 043import org.junit.ClassRule; 044import org.junit.Rule; 045import org.junit.Test; 046import org.junit.experimental.categories.Category; 047import org.junit.rules.TestName; 048import org.slf4j.Logger; 049import org.slf4j.LoggerFactory; 050 051/** 052 * Test that the snapshot hfile cleaner finds hfiles referenced in a snapshot 053 */ 054@Category({MasterTests.class, SmallTests.class}) 055public class TestSnapshotHFileCleaner { 056 057 @ClassRule 058 public static final HBaseClassTestRule CLASS_RULE = 059 HBaseClassTestRule.forClass(TestSnapshotHFileCleaner.class); 060 061 private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFileCache.class); 062 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 063 private static final String TABLE_NAME_STR = "testSnapshotManifest"; 064 private static final String SNAPSHOT_NAME_STR = "testSnapshotManifest-snapshot"; 065 private static Path rootDir; 066 private static FileSystem fs; 067 068 @Rule 069 public TestName name = new TestName(); 070 071 /** 072 * Setup the test environment 073 */ 074 @BeforeClass 075 public static void setup() throws Exception { 076 Configuration conf = TEST_UTIL.getConfiguration(); 077 rootDir = FSUtils.getRootDir(conf); 078 fs = FileSystem.get(conf); 079 } 080 081 082 @AfterClass 083 public static void cleanup() throws IOException { 084 // cleanup 085 fs.delete(rootDir, true); 086 } 087 088 @Test 089 public void testFindsSnapshotFilesWhenCleaning() throws IOException { 090 Configuration conf = TEST_UTIL.getConfiguration(); 091 FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); 092 Path rootDir = FSUtils.getRootDir(conf); 093 Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY); 094 095 FileSystem fs = FileSystem.get(conf); 096 SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner(); 097 cleaner.setConf(conf); 098 099 // write an hfile to the snapshot directory 100 String snapshotName = "snapshot"; 101 byte[] snapshot = Bytes.toBytes(snapshotName); 102 final TableName tableName = TableName.valueOf(name.getMethodName()); 103 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); 104 HRegionInfo mockRegion = new HRegionInfo(tableName); 105 Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName()); 106 Path familyDir = new Path(regionSnapshotDir, "family"); 107 // create a reference to a supposedly valid hfile 108 String hfile = "fd1e73e8a96c486090c5cec07b4894c4"; 109 Path refFile = new Path(familyDir, hfile); 110 111 // make sure the reference file exists 112 fs.create(refFile); 113 114 // create the hfile in the archive 115 fs.mkdirs(archivedHfileDir); 116 fs.createNewFile(new Path(archivedHfileDir, hfile)); 117 118 // make sure that the file isn't deletable 119 assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile))); 120 } 121 122 static class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector { 123 @Override 124 public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException { 125 Collection<String> files = new HashSet<>(); 126 files.addAll(SnapshotReferenceUtil.getHFileNames(TEST_UTIL.getConfiguration(), fs, snapshotDir)); 127 return files; 128 } 129 } 130 131 /** 132 * If there is a corrupted region manifest, it should throw out CorruptedSnapshotException, 133 * instead of an IOException 134 */ 135 @Test 136 public void testCorruptedRegionManifest() throws IOException { 137 SnapshotTestingUtils.SnapshotMock 138 snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); 139 SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( 140 SNAPSHOT_NAME_STR, TABLE_NAME_STR); 141 builder.addRegionV2(); 142 builder.corruptOneRegionManifest(); 143 144 long period = Long.MAX_VALUE; 145 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, 146 "test-snapshot-file-cache-refresh", new SnapshotFiles()); 147 try { 148 cache.getSnapshotsInProgress(); 149 } catch (CorruptedSnapshotException cse) { 150 LOG.info("Expected exception " + cse); 151 } finally { 152 fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true); 153 } 154 } 155 156 /** 157 * If there is a corrupted data manifest, it should throw out CorruptedSnapshotException, 158 * instead of an IOException 159 */ 160 @Test 161 public void testCorruptedDataManifest() throws IOException { 162 SnapshotTestingUtils.SnapshotMock 163 snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); 164 SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( 165 SNAPSHOT_NAME_STR, TABLE_NAME_STR); 166 builder.addRegionV2(); 167 // consolidate to generate a data.manifest file 168 builder.consolidate(); 169 builder.corruptDataManifest(); 170 171 long period = Long.MAX_VALUE; 172 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, 173 "test-snapshot-file-cache-refresh", new SnapshotFiles()); 174 try { 175 cache.getSnapshotsInProgress(); 176 } catch (CorruptedSnapshotException cse) { 177 LOG.info("Expected exception " + cse); 178 } finally { 179 fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true); 180 } 181 } 182 183 /** 184 * HBASE-16464 185 */ 186 @Test 187 public void testMissedTmpSnapshot() throws IOException { 188 SnapshotTestingUtils.SnapshotMock 189 snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); 190 SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( 191 SNAPSHOT_NAME_STR, TABLE_NAME_STR); 192 builder.addRegionV2(); 193 builder.missOneRegionSnapshotFile(); 194 195 long period = Long.MAX_VALUE; 196 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, 197 "test-snapshot-file-cache-refresh", new SnapshotFiles()); 198 cache.getSnapshotsInProgress(); 199 assertFalse(fs.exists(builder.getSnapshotsDir())); 200 } 201}