001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.cleaner; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertTrue; 023 024import java.io.IOException; 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.fs.FileStatus; 027import org.apache.hadoop.fs.FileSystem; 028import org.apache.hadoop.fs.Path; 029import org.apache.hadoop.hbase.ChoreService; 030import org.apache.hadoop.hbase.CoordinatedStateManager; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HBaseTestingUtility; 033import org.apache.hadoop.hbase.Server; 034import org.apache.hadoop.hbase.ServerName; 035import org.apache.hadoop.hbase.TableName; 036import org.apache.hadoop.hbase.client.ClusterConnection; 037import org.apache.hadoop.hbase.client.Connection; 038import org.apache.hadoop.hbase.client.RegionInfo; 039import org.apache.hadoop.hbase.client.RegionInfoBuilder; 040import org.apache.hadoop.hbase.io.HFileLink; 041import org.apache.hadoop.hbase.testclassification.MasterTests; 042import org.apache.hadoop.hbase.testclassification.MediumTests; 043import org.apache.hadoop.hbase.util.FSUtils; 044import org.apache.hadoop.hbase.util.HFileArchiveUtil; 045import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; 046import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 047import org.junit.AfterClass; 048import org.junit.BeforeClass; 049import org.junit.ClassRule; 050import org.junit.Rule; 051import org.junit.Test; 052import org.junit.experimental.categories.Category; 053import org.junit.rules.TestName; 054 055/** 056 * Test the HFileLink Cleaner. HFiles with links cannot be deleted until a link is present. 057 */ 058@Category({ MasterTests.class, MediumTests.class }) 059public class TestHFileLinkCleaner { 060 061 @ClassRule 062 public static final HBaseClassTestRule CLASS_RULE = 063 HBaseClassTestRule.forClass(TestHFileLinkCleaner.class); 064 065 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 066 067 private static DirScanPool POOL; 068 069 @Rule 070 public TestName name = new TestName(); 071 072 @BeforeClass 073 public static void setUp() { 074 POOL = new DirScanPool(TEST_UTIL.getConfiguration()); 075 } 076 077 @AfterClass 078 public static void tearDown() { 079 POOL.shutdownNow(); 080 } 081 082 @Test 083 public void testHFileLinkCleaning() throws Exception { 084 Configuration conf = TEST_UTIL.getConfiguration(); 085 FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); 086 conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, HFileLinkCleaner.class.getName()); 087 Path rootDir = FSUtils.getRootDir(conf); 088 FileSystem fs = FileSystem.get(conf); 089 090 final TableName tableName = TableName.valueOf(name.getMethodName()); 091 final TableName tableLinkName = TableName.valueOf(name.getMethodName() + "-link"); 092 final String hfileName = "1234567890"; 093 final String familyName = "cf"; 094 095 RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build(); 096 RegionInfo hriLink = RegionInfoBuilder.newBuilder(tableLinkName).build(); 097 098 Path archiveDir = HFileArchiveUtil.getArchivePath(conf); 099 Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, 100 tableName, hri.getEncodedName(), familyName); 101 102 // Create hfile /hbase/table-link/region/cf/getEncodedName.HFILE(conf); 103 Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName); 104 fs.mkdirs(familyPath); 105 Path hfilePath = new Path(familyPath, hfileName); 106 fs.createNewFile(hfilePath); 107 108 // Create link to hfile 109 Path familyLinkPath = 110 getFamilyDirPath(rootDir, tableLinkName, hriLink.getEncodedName(), familyName); 111 fs.mkdirs(familyLinkPath); 112 HFileLink.create(conf, fs, familyLinkPath, hri, hfileName); 113 Path linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName); 114 assertTrue(fs.exists(linkBackRefDir)); 115 FileStatus[] backRefs = fs.listStatus(linkBackRefDir); 116 assertEquals(1, backRefs.length); 117 Path linkBackRef = backRefs[0].getPath(); 118 119 // Initialize cleaner 120 final long ttl = 1000; 121 conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl); 122 Server server = new DummyServer(); 123 HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir, POOL); 124 125 // Link backref cannot be removed 126 cleaner.chore(); 127 assertTrue(fs.exists(linkBackRef)); 128 assertTrue(fs.exists(hfilePath)); 129 130 // Link backref can be removed 131 fs.rename(FSUtils.getTableDir(rootDir, tableLinkName), 132 FSUtils.getTableDir(archiveDir, tableLinkName)); 133 cleaner.chore(); 134 assertFalse("Link should be deleted", fs.exists(linkBackRef)); 135 136 // HFile can be removed 137 Thread.sleep(ttl * 2); 138 cleaner.chore(); 139 assertFalse("HFile should be deleted", fs.exists(hfilePath)); 140 141 // Remove everything 142 for (int i = 0; i < 4; ++i) { 143 Thread.sleep(ttl * 2); 144 cleaner.chore(); 145 } 146 assertFalse("HFile should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableName))); 147 assertFalse("Link should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableLinkName))); 148 } 149 150 private static Path getFamilyDirPath (final Path rootDir, final TableName table, 151 final String region, final String family) { 152 return new Path(new Path(FSUtils.getTableDir(rootDir, table), region), family); 153 } 154 155 static class DummyServer implements Server { 156 157 @Override 158 public Configuration getConfiguration() { 159 return TEST_UTIL.getConfiguration(); 160 } 161 162 @Override 163 public ZKWatcher getZooKeeper() { 164 try { 165 return new ZKWatcher(getConfiguration(), "dummy server", this); 166 } catch (IOException e) { 167 e.printStackTrace(); 168 } 169 return null; 170 } 171 172 @Override 173 public CoordinatedStateManager getCoordinatedStateManager() { 174 return null; 175 } 176 177 @Override 178 public ClusterConnection getConnection() { 179 return null; 180 } 181 182 @Override 183 public MetaTableLocator getMetaTableLocator() { 184 return null; 185 } 186 187 @Override 188 public ServerName getServerName() { 189 return ServerName.valueOf("regionserver,60020,000000"); 190 } 191 192 @Override 193 public void abort(String why, Throwable e) {} 194 195 @Override 196 public boolean isAborted() { 197 return false; 198 } 199 200 @Override 201 public void stop(String why) {} 202 203 @Override 204 public boolean isStopped() { 205 return false; 206 } 207 208 @Override 209 public ChoreService getChoreService() { 210 return null; 211 } 212 213 @Override 214 public ClusterConnection getClusterConnection() { 215 // TODO Auto-generated method stub 216 return null; 217 } 218 219 @Override 220 public FileSystem getFileSystem() { 221 return null; 222 } 223 224 @Override 225 public boolean isStopping() { 226 return false; 227 } 228 229 @Override 230 public Connection createConnection(Configuration conf) throws IOException { 231 return null; 232 } 233 } 234}