001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.snapshot; 019 020import static org.junit.Assert.assertFalse; 021import static org.junit.Assert.assertTrue; 022 023import java.io.IOException; 024import java.util.ArrayList; 025import java.util.Arrays; 026import java.util.Collection; 027import java.util.HashSet; 028import java.util.List; 029import org.apache.hadoop.fs.FileStatus; 030import org.apache.hadoop.fs.FileSystem; 031import org.apache.hadoop.fs.Path; 032import org.apache.hadoop.hbase.HBaseClassTestRule; 033import org.apache.hadoop.hbase.HBaseTestingUtility; 034import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; 035import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; 036import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; 037import org.apache.hadoop.hbase.testclassification.MasterTests; 038import org.apache.hadoop.hbase.testclassification.MediumTests; 039import org.apache.hadoop.hbase.util.FSUtils; 040import org.junit.After; 041import org.junit.AfterClass; 042import org.junit.BeforeClass; 043import org.junit.ClassRule; 044import org.junit.Test; 045import org.junit.experimental.categories.Category; 046import org.slf4j.Logger; 047import org.slf4j.LoggerFactory; 048 049/** 050 * Test that we correctly reload the cache, filter directories, etc. 051 */ 052@Category({MasterTests.class, MediumTests.class}) 053public class TestSnapshotFileCache { 054 055 @ClassRule 056 public static final HBaseClassTestRule CLASS_RULE = 057 HBaseClassTestRule.forClass(TestSnapshotFileCache.class); 058 059 private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFileCache.class); 060 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); 061 private static FileSystem fs; 062 private static Path rootDir; 063 064 @BeforeClass 065 public static void startCluster() throws Exception { 066 UTIL.startMiniDFSCluster(1); 067 fs = UTIL.getDFSCluster().getFileSystem(); 068 rootDir = UTIL.getDefaultRootDirPath(); 069 } 070 071 @AfterClass 072 public static void stopCluster() throws Exception { 073 UTIL.shutdownMiniDFSCluster(); 074 } 075 076 @After 077 public void cleanupFiles() throws Exception { 078 // cleanup the snapshot directory 079 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); 080 fs.delete(snapshotDir, true); 081 } 082 083 @Test 084 public void testLoadAndDelete() throws IOException { 085 // don't refresh the cache unless we tell it to 086 long period = Long.MAX_VALUE; 087 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, 088 "test-snapshot-file-cache-refresh", new SnapshotFiles()); 089 090 createAndTestSnapshotV1(cache, "snapshot1a", false, true); 091 092 createAndTestSnapshotV2(cache, "snapshot2a", false, true); 093 } 094 095 @Test 096 public void testReloadModifiedDirectory() throws IOException { 097 // don't refresh the cache unless we tell it to 098 long period = Long.MAX_VALUE; 099 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, 100 "test-snapshot-file-cache-refresh", new SnapshotFiles()); 101 102 createAndTestSnapshotV1(cache, "snapshot1", false, true); 103 // now delete the snapshot and add a file with a different name 104 createAndTestSnapshotV1(cache, "snapshot1", false, false); 105 106 createAndTestSnapshotV2(cache, "snapshot2", false, true); 107 // now delete the snapshot and add a file with a different name 108 createAndTestSnapshotV2(cache, "snapshot2", false, false); 109 } 110 111 @Test 112 public void testSnapshotTempDirReload() throws IOException { 113 long period = Long.MAX_VALUE; 114 // This doesn't refresh cache until we invoke it explicitly 115 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000, 116 "test-snapshot-file-cache-refresh", new SnapshotFiles()); 117 118 // Add a new non-tmp snapshot 119 createAndTestSnapshotV1(cache, "snapshot0v1", false, false); 120 createAndTestSnapshotV1(cache, "snapshot0v2", false, false); 121 } 122 123 class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector { 124 @Override 125 public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException { 126 Collection<String> files = new HashSet<>(); 127 files.addAll(SnapshotReferenceUtil.getHFileNames(UTIL.getConfiguration(), fs, snapshotDir)); 128 return files; 129 } 130 }; 131 132 private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache, 133 final String name, final boolean tmp, final boolean removeOnExit) throws IOException { 134 SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); 135 SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name, name); 136 createAndTestSnapshot(cache, builder, tmp, removeOnExit); 137 return builder; 138 } 139 140 private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name, 141 final boolean tmp, final boolean removeOnExit) throws IOException { 142 SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); 143 SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name, name); 144 createAndTestSnapshot(cache, builder, tmp, removeOnExit); 145 } 146 147 private void createAndTestSnapshot(final SnapshotFileCache cache, 148 final SnapshotMock.SnapshotBuilder builder, 149 final boolean tmp, final boolean removeOnExit) throws IOException { 150 List<Path> files = new ArrayList<>(); 151 for (int i = 0; i < 3; ++i) { 152 for (Path filePath: builder.addRegion()) { 153 files.add(filePath); 154 } 155 } 156 157 // Finalize the snapshot 158 builder.commit(); 159 160 // Make sure that all files are still present 161 for (Path path: files) { 162 assertFalse("Cache didn't find " + path, contains(getNonSnapshotFiles(cache, path), path)); 163 } 164 165 FSUtils.logFileSystemState(fs, rootDir, LOG); 166 if (removeOnExit) { 167 LOG.debug("Deleting snapshot."); 168 fs.delete(builder.getSnapshotsDir(), true); 169 FSUtils.logFileSystemState(fs, rootDir, LOG); 170 171 // then trigger a refresh 172 cache.triggerCacheRefreshForTesting(); 173 // and not it shouldn't find those files 174 for (Path filePath: files) { 175 assertTrue("Cache found '" + filePath + "', but it shouldn't have.", 176 contains(getNonSnapshotFiles(cache, filePath), filePath)); 177 178 } 179 } 180 } 181 182 private static boolean contains(Iterable<FileStatus> files, Path filePath) { 183 for (FileStatus status: files) { 184 LOG.debug("debug in contains, 3.1: " + status.getPath() + " filePath:" + filePath); 185 if (filePath.equals(status.getPath())) { 186 return true; 187 } 188 } 189 return false; 190 } 191 192 private static Iterable<FileStatus> getNonSnapshotFiles(SnapshotFileCache cache, Path storeFile) 193 throws IOException { 194 return cache.getUnreferencedFiles( 195 Arrays.asList(FSUtils.listStatus(fs, storeFile.getParent())), null 196 ); 197 } 198}