001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.snapshot; 019 020import static org.junit.Assert.assertFalse; 021 022import java.io.IOException; 023import java.util.Collection; 024import java.util.HashSet; 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.fs.FileSystem; 027import org.apache.hadoop.fs.Path; 028import org.apache.hadoop.hbase.HBaseClassTestRule; 029import org.apache.hadoop.hbase.HBaseTestingUtility; 030import org.apache.hadoop.hbase.HConstants; 031import org.apache.hadoop.hbase.HRegionInfo; 032import org.apache.hadoop.hbase.TableName; 033import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; 034import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; 035import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; 036import org.apache.hadoop.hbase.testclassification.MasterTests; 037import org.apache.hadoop.hbase.testclassification.SmallTests; 038import org.apache.hadoop.hbase.util.Bytes; 039import org.apache.hadoop.hbase.util.FSUtils; 040import org.junit.AfterClass; 041import org.junit.BeforeClass; 042import org.junit.ClassRule; 043import org.junit.Rule; 044import org.junit.Test; 045import org.junit.experimental.categories.Category; 046import org.junit.rules.TestName; 047import org.slf4j.Logger; 048import org.slf4j.LoggerFactory; 049 050/** 051 * Test that the snapshot hfile cleaner finds hfiles referenced in a snapshot 052 */ 053@Category({MasterTests.class, SmallTests.class}) 054public class TestSnapshotHFileCleaner { 055 056 @ClassRule 057 public static final HBaseClassTestRule CLASS_RULE = 058 HBaseClassTestRule.forClass(TestSnapshotHFileCleaner.class); 059 060 private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFileCache.class); 061 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 062 private static final String TABLE_NAME_STR = "testSnapshotManifest"; 063 private static final String SNAPSHOT_NAME_STR = "testSnapshotManifest-snapshot"; 064 private static Path rootDir; 065 private static FileSystem fs; 066 067 @Rule 068 public TestName name = new TestName(); 069 070 /** 071 * Setup the test environment 072 */ 073 @BeforeClass 074 public static void setup() throws Exception { 075 Configuration conf = TEST_UTIL.getConfiguration(); 076 rootDir = FSUtils.getRootDir(conf); 077 fs = FileSystem.get(conf); 078 } 079 080 081 @AfterClass 082 public static void cleanup() throws IOException { 083 // cleanup 084 fs.delete(rootDir, true); 085 } 086 087 @Test 088 public void testFindsSnapshotFilesWhenCleaning() throws IOException { 089 Configuration conf = TEST_UTIL.getConfiguration(); 090 FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); 091 Path rootDir = FSUtils.getRootDir(conf); 092 Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY); 093 094 FileSystem fs = FileSystem.get(conf); 095 SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner(); 096 cleaner.setConf(conf); 097 098 // write an hfile to the snapshot directory 099 String snapshotName = "snapshot"; 100 byte[] snapshot = Bytes.toBytes(snapshotName); 101 final TableName tableName = TableName.valueOf(name.getMethodName()); 102 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); 103 HRegionInfo mockRegion = new HRegionInfo(tableName); 104 Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName()); 105 Path familyDir = new Path(regionSnapshotDir, "family"); 106 // create a reference to a supposedly valid hfile 107 String hfile = "fd1e73e8a96c486090c5cec07b4894c4"; 108 Path refFile = new Path(familyDir, hfile); 109 110 // make sure the reference file exists 111 fs.create(refFile); 112 113 // create the hfile in the archive 114 fs.mkdirs(archivedHfileDir); 115 fs.createNewFile(new Path(archivedHfileDir, hfile)); 116 117 // make sure that the file isn't deletable 118 assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile))); 119 } 120 121 static class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector { 122 @Override 123 public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException { 124 Collection<String> files = new HashSet<>(); 125 files.addAll(SnapshotReferenceUtil.getHFileNames(TEST_UTIL.getConfiguration(), fs, snapshotDir)); 126 return files; 127 } 128 } 129 130 /** 131 * If there is a corrupted region manifest, it should throw out CorruptedSnapshotException, 132 * instead of an IOException 133 */ 134 @Test 135 public void testCorruptedRegionManifest() throws IOException { 136 SnapshotTestingUtils.SnapshotMock 137 snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); 138 SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( 139 SNAPSHOT_NAME_STR, TABLE_NAME_STR); 140 builder.addRegionV2(); 141 builder.corruptOneRegionManifest(); 142 143 fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, TEST_UTIL.getConfiguration()), 144 true); 145 } 146 147 /** 148 * If there is a corrupted data manifest, it should throw out CorruptedSnapshotException, 149 * instead of an IOException 150 */ 151 @Test 152 public void testCorruptedDataManifest() throws IOException { 153 SnapshotTestingUtils.SnapshotMock 154 snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); 155 SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2( 156 SNAPSHOT_NAME_STR, TABLE_NAME_STR); 157 builder.addRegionV2(); 158 // consolidate to generate a data.manifest file 159 builder.consolidate(); 160 builder.corruptDataManifest(); 161 162 fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, 163 TEST_UTIL.getConfiguration()), true); 164 } 165}