001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.cleaner; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertTrue; 023 024import java.io.IOException; 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.fs.FileStatus; 027import org.apache.hadoop.fs.FileSystem; 028import org.apache.hadoop.fs.Path; 029import org.apache.hadoop.hbase.ChoreService; 030import org.apache.hadoop.hbase.CoordinatedStateManager; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HBaseTestingUtility; 033import org.apache.hadoop.hbase.Server; 034import org.apache.hadoop.hbase.ServerName; 035import org.apache.hadoop.hbase.TableName; 036import org.apache.hadoop.hbase.client.ClusterConnection; 037import org.apache.hadoop.hbase.client.Connection; 038import org.apache.hadoop.hbase.client.RegionInfo; 039import org.apache.hadoop.hbase.client.RegionInfoBuilder; 040import org.apache.hadoop.hbase.io.HFileLink; 041import org.apache.hadoop.hbase.testclassification.MasterTests; 042import org.apache.hadoop.hbase.testclassification.MediumTests; 043import org.apache.hadoop.hbase.util.FSUtils; 044import org.apache.hadoop.hbase.util.HFileArchiveUtil; 045import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 046import org.junit.AfterClass; 047import org.junit.BeforeClass; 048import org.junit.ClassRule; 049import org.junit.Rule; 050import org.junit.Test; 051import org.junit.experimental.categories.Category; 052import org.junit.rules.TestName; 053 054/** 055 * Test the HFileLink Cleaner. HFiles with links cannot be deleted until a link is present. 056 */ 057@Category({ MasterTests.class, MediumTests.class }) 058public class TestHFileLinkCleaner { 059 060 @ClassRule 061 public static final HBaseClassTestRule CLASS_RULE = 062 HBaseClassTestRule.forClass(TestHFileLinkCleaner.class); 063 064 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 065 066 private static DirScanPool POOL; 067 068 @Rule 069 public TestName name = new TestName(); 070 071 @BeforeClass 072 public static void setUp() { 073 POOL = new DirScanPool(TEST_UTIL.getConfiguration()); 074 } 075 076 @AfterClass 077 public static void tearDown() { 078 POOL.shutdownNow(); 079 } 080 081 @Test 082 public void testHFileLinkCleaning() throws Exception { 083 Configuration conf = TEST_UTIL.getConfiguration(); 084 FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); 085 conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, HFileLinkCleaner.class.getName()); 086 Path rootDir = FSUtils.getRootDir(conf); 087 FileSystem fs = FileSystem.get(conf); 088 089 final TableName tableName = TableName.valueOf(name.getMethodName()); 090 final TableName tableLinkName = TableName.valueOf(name.getMethodName() + "-link"); 091 final String hfileName = "1234567890"; 092 final String familyName = "cf"; 093 094 RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build(); 095 RegionInfo hriLink = RegionInfoBuilder.newBuilder(tableLinkName).build(); 096 097 Path archiveDir = HFileArchiveUtil.getArchivePath(conf); 098 Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, 099 tableName, hri.getEncodedName(), familyName); 100 101 // Create hfile /hbase/table-link/region/cf/getEncodedName.HFILE(conf); 102 Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName); 103 fs.mkdirs(familyPath); 104 Path hfilePath = new Path(familyPath, hfileName); 105 fs.createNewFile(hfilePath); 106 107 // Create link to hfile 108 Path familyLinkPath = 109 getFamilyDirPath(rootDir, tableLinkName, hriLink.getEncodedName(), familyName); 110 fs.mkdirs(familyLinkPath); 111 HFileLink.create(conf, fs, familyLinkPath, hri, hfileName); 112 Path linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName); 113 assertTrue(fs.exists(linkBackRefDir)); 114 FileStatus[] backRefs = fs.listStatus(linkBackRefDir); 115 assertEquals(1, backRefs.length); 116 Path linkBackRef = backRefs[0].getPath(); 117 118 // Initialize cleaner 119 final long ttl = 1000; 120 conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl); 121 Server server = new DummyServer(); 122 HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir, POOL); 123 124 // Link backref cannot be removed 125 cleaner.chore(); 126 assertTrue(fs.exists(linkBackRef)); 127 assertTrue(fs.exists(hfilePath)); 128 129 // Link backref can be removed 130 fs.rename(FSUtils.getTableDir(rootDir, tableLinkName), 131 FSUtils.getTableDir(archiveDir, tableLinkName)); 132 cleaner.chore(); 133 assertFalse("Link should be deleted", fs.exists(linkBackRef)); 134 135 // HFile can be removed 136 Thread.sleep(ttl * 2); 137 cleaner.chore(); 138 assertFalse("HFile should be deleted", fs.exists(hfilePath)); 139 140 // Remove everything 141 for (int i = 0; i < 4; ++i) { 142 Thread.sleep(ttl * 2); 143 cleaner.chore(); 144 } 145 assertFalse("HFile should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableName))); 146 assertFalse("Link should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableLinkName))); 147 } 148 149 private static Path getFamilyDirPath (final Path rootDir, final TableName table, 150 final String region, final String family) { 151 return new Path(new Path(FSUtils.getTableDir(rootDir, table), region), family); 152 } 153 154 static class DummyServer implements Server { 155 156 @Override 157 public Configuration getConfiguration() { 158 return TEST_UTIL.getConfiguration(); 159 } 160 161 @Override 162 public ZKWatcher getZooKeeper() { 163 try { 164 return new ZKWatcher(getConfiguration(), "dummy server", this); 165 } catch (IOException e) { 166 e.printStackTrace(); 167 } 168 return null; 169 } 170 171 @Override 172 public CoordinatedStateManager getCoordinatedStateManager() { 173 return null; 174 } 175 176 @Override 177 public ClusterConnection getConnection() { 178 return null; 179 } 180 181 @Override 182 public ServerName getServerName() { 183 return ServerName.valueOf("regionserver,60020,000000"); 184 } 185 186 @Override 187 public void abort(String why, Throwable e) {} 188 189 @Override 190 public boolean isAborted() { 191 return false; 192 } 193 194 @Override 195 public void stop(String why) {} 196 197 @Override 198 public boolean isStopped() { 199 return false; 200 } 201 202 @Override 203 public ChoreService getChoreService() { 204 return null; 205 } 206 207 @Override 208 public ClusterConnection getClusterConnection() { 209 // TODO Auto-generated method stub 210 return null; 211 } 212 213 @Override 214 public FileSystem getFileSystem() { 215 return null; 216 } 217 218 @Override 219 public boolean isStopping() { 220 return false; 221 } 222 223 @Override 224 public Connection createConnection(Configuration conf) throws IOException { 225 return null; 226 } 227 } 228}