001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.snapshot;
019
020import static org.junit.Assert.assertFalse;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Arrays;
026import java.util.Collection;
027import java.util.HashSet;
028import java.util.List;
029import org.apache.hadoop.fs.FileStatus;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtility;
034import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
035import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
036import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
037import org.apache.hadoop.hbase.testclassification.MasterTests;
038import org.apache.hadoop.hbase.testclassification.MediumTests;
039import org.apache.hadoop.hbase.util.FSUtils;
040import org.junit.After;
041import org.junit.AfterClass;
042import org.junit.BeforeClass;
043import org.junit.ClassRule;
044import org.junit.Test;
045import org.junit.experimental.categories.Category;
046import org.slf4j.Logger;
047import org.slf4j.LoggerFactory;
048
049/**
050 * Test that we correctly reload the cache, filter directories, etc.
051 */
052@Category({MasterTests.class, MediumTests.class})
053public class TestSnapshotFileCache {
054
055  @ClassRule
056  public static final HBaseClassTestRule CLASS_RULE =
057      HBaseClassTestRule.forClass(TestSnapshotFileCache.class);
058
059  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFileCache.class);
060  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
061  private static FileSystem fs;
062  private static Path rootDir;
063
064  @BeforeClass
065  public static void startCluster() throws Exception {
066    UTIL.startMiniDFSCluster(1);
067    fs = UTIL.getDFSCluster().getFileSystem();
068    rootDir = UTIL.getDefaultRootDirPath();
069  }
070
071  @AfterClass
072  public static void stopCluster() throws Exception {
073    UTIL.shutdownMiniDFSCluster();
074  }
075
076  @After
077  public void cleanupFiles() throws Exception {
078    // cleanup the snapshot directory
079    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
080    fs.delete(snapshotDir, true);
081  }
082
083  @Test
084  public void testLoadAndDelete() throws IOException {
085    // don't refresh the cache unless we tell it to
086    long period = Long.MAX_VALUE;
087    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
088        "test-snapshot-file-cache-refresh", new SnapshotFiles());
089
090    createAndTestSnapshotV1(cache, "snapshot1a", false, true);
091
092    createAndTestSnapshotV2(cache, "snapshot2a", false, true);
093  }
094
095  @Test
096  public void testReloadModifiedDirectory() throws IOException {
097    // don't refresh the cache unless we tell it to
098    long period = Long.MAX_VALUE;
099    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
100        "test-snapshot-file-cache-refresh", new SnapshotFiles());
101
102    createAndTestSnapshotV1(cache, "snapshot1", false, true);
103    // now delete the snapshot and add a file with a different name
104    createAndTestSnapshotV1(cache, "snapshot1", false, false);
105
106    createAndTestSnapshotV2(cache, "snapshot2", false, true);
107    // now delete the snapshot and add a file with a different name
108    createAndTestSnapshotV2(cache, "snapshot2", false, false);
109  }
110
111  @Test
112  public void testSnapshotTempDirReload() throws IOException {
113    long period = Long.MAX_VALUE;
114    // This doesn't refresh cache until we invoke it explicitly
115    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
116        "test-snapshot-file-cache-refresh", new SnapshotFiles());
117
118    // Add a new non-tmp snapshot
119    createAndTestSnapshotV1(cache, "snapshot0v1", false, false);
120    createAndTestSnapshotV1(cache, "snapshot0v2", false, false);
121  }
122
123  class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
124    @Override
125    public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
126      Collection<String> files =  new HashSet<>();
127      files.addAll(SnapshotReferenceUtil.getHFileNames(UTIL.getConfiguration(), fs, snapshotDir));
128      return files;
129    }
130  };
131
132  private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache,
133      final String name, final boolean tmp, final boolean removeOnExit) throws IOException {
134    SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
135    SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name, name);
136    createAndTestSnapshot(cache, builder, tmp, removeOnExit);
137    return builder;
138  }
139
140  private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name,
141      final boolean tmp, final boolean removeOnExit) throws IOException {
142    SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
143    SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name, name);
144    createAndTestSnapshot(cache, builder, tmp, removeOnExit);
145  }
146
147  private void createAndTestSnapshot(final SnapshotFileCache cache,
148      final SnapshotMock.SnapshotBuilder builder,
149      final boolean tmp, final boolean removeOnExit) throws IOException {
150    List<Path> files = new ArrayList<>();
151    for (int i = 0; i < 3; ++i) {
152      for (Path filePath: builder.addRegion()) {
153        files.add(filePath);
154      }
155    }
156
157    // Finalize the snapshot
158    builder.commit();
159
160    // Make sure that all files are still present
161    for (Path path: files) {
162      assertFalse("Cache didn't find " + path, contains(getNonSnapshotFiles(cache, path), path));
163    }
164
165    FSUtils.logFileSystemState(fs, rootDir, LOG);
166    if (removeOnExit) {
167      LOG.debug("Deleting snapshot.");
168      fs.delete(builder.getSnapshotsDir(), true);
169      FSUtils.logFileSystemState(fs, rootDir, LOG);
170
171      // then trigger a refresh
172      cache.triggerCacheRefreshForTesting();
173      // and not it shouldn't find those files
174      for (Path filePath: files) {
175        assertTrue("Cache found '" + filePath + "', but it shouldn't have.",
176          contains(getNonSnapshotFiles(cache, filePath), filePath));
177
178      }
179    }
180  }
181
182  private static boolean contains(Iterable<FileStatus> files, Path filePath) {
183    for (FileStatus status: files) {
184      LOG.debug("debug in contains, 3.1: " + status.getPath() + " filePath:" + filePath);
185      if (filePath.equals(status.getPath())) {
186        return true;
187      }
188    }
189    return false;
190  }
191
192  private static Iterable<FileStatus> getNonSnapshotFiles(SnapshotFileCache cache, Path storeFile)
193      throws IOException {
194    return cache.getUnreferencedFiles(
195        Arrays.asList(FSUtils.listStatus(fs, storeFile.getParent())), null
196    );
197  }
198}