001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024import static org.mockito.ArgumentMatchers.any;
025import static org.mockito.Mockito.mock;
026import static org.mockito.Mockito.never;
027import static org.mockito.Mockito.times;
028import static org.mockito.Mockito.verify;
029import static org.mockito.Mockito.when;
030
031import java.io.IOException;
032import java.security.PrivilegedExceptionAction;
033import java.util.ArrayList;
034import java.util.Collection;
035import java.util.Collections;
036import java.util.HashMap;
037import java.util.List;
038import java.util.stream.Collectors;
039import org.apache.hadoop.conf.Configuration;
040import org.apache.hadoop.fs.FileStatus;
041import org.apache.hadoop.fs.FileSystem;
042import org.apache.hadoop.fs.Path;
043import org.apache.hadoop.fs.PathFilter;
044import org.apache.hadoop.hbase.ChoreService;
045import org.apache.hadoop.hbase.HBaseClassTestRule;
046import org.apache.hadoop.hbase.HBaseTestingUtility;
047import org.apache.hadoop.hbase.HConstants;
048import org.apache.hadoop.hbase.Stoppable;
049import org.apache.hadoop.hbase.TableName;
050import org.apache.hadoop.hbase.client.Admin;
051import org.apache.hadoop.hbase.client.RegionInfo;
052import org.apache.hadoop.hbase.client.Table;
053import org.apache.hadoop.hbase.master.cleaner.DirScanPool;
054import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
055import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
056import org.apache.hadoop.hbase.regionserver.HRegion;
057import org.apache.hadoop.hbase.regionserver.HRegionServer;
058import org.apache.hadoop.hbase.regionserver.HStoreFile;
059import org.apache.hadoop.hbase.testclassification.LargeTests;
060import org.apache.hadoop.hbase.testclassification.MiscTests;
061import org.apache.hadoop.hbase.util.Bytes;
062import org.apache.hadoop.hbase.util.CommonFSUtils;
063import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
064import org.apache.hadoop.hbase.util.FSUtils;
065import org.apache.hadoop.hbase.util.HFileArchiveTestingUtil;
066import org.apache.hadoop.hbase.util.HFileArchiveUtil;
067import org.apache.hadoop.hbase.util.StoppableImplementation;
068import org.apache.hadoop.security.UserGroupInformation;
069import org.junit.After;
070import org.junit.AfterClass;
071import org.junit.Assert;
072import org.junit.BeforeClass;
073import org.junit.ClassRule;
074import org.junit.Rule;
075import org.junit.Test;
076import org.junit.experimental.categories.Category;
077import org.junit.rules.TestName;
078import org.mockito.ArgumentCaptor;
079import org.slf4j.Logger;
080import org.slf4j.LoggerFactory;
081
082/**
083 * Test that the {@link HFileArchiver} correctly removes all the parts of a region when cleaning up
084 * a region
085 */
086@Category({ LargeTests.class, MiscTests.class })
087public class TestHFileArchiving {
088
089  @ClassRule
090  public static final HBaseClassTestRule CLASS_RULE =
091    HBaseClassTestRule.forClass(TestHFileArchiving.class);
092
093  private static final Logger LOG = LoggerFactory.getLogger(TestHFileArchiving.class);
094  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
095  private static final byte[] TEST_FAM = Bytes.toBytes("fam");
096
097  private static DirScanPool POOL;
098  @Rule
099  public TestName name = new TestName();
100
101  /**
102   * Setup the config for the cluster
103   */
104  @BeforeClass
105  public static void setupCluster() throws Exception {
106    setupConf(UTIL.getConfiguration());
107    UTIL.startMiniCluster();
108
109    // We don't want the cleaner to remove files. The tests do that.
110    UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().cancel(true);
111
112    POOL = DirScanPool.getHFileCleanerScanPool(UTIL.getConfiguration());
113  }
114
115  private static void setupConf(Configuration conf) {
116    // disable the ui
117    conf.setInt("hbase.regionsever.info.port", -1);
118    // drop the memstore size so we get flushes
119    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
120    // disable major compactions
121    conf.setInt(HConstants.MAJOR_COMPACTION_PERIOD, 0);
122
123    // prevent aggressive region split
124    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
125      ConstantSizeRegionSplitPolicy.class.getName());
126  }
127
128  @After
129  public void tearDown() throws Exception {
130    // cleanup the archive directory
131    clearArchiveDirectory();
132  }
133
134  @AfterClass
135  public static void cleanupTest() throws Exception {
136    UTIL.shutdownMiniCluster();
137    POOL.shutdownNow();
138  }
139
140  @Test
141  public void testArchiveStoreFilesDifferentFileSystemsWallWithSchemaPlainRoot() throws Exception {
142    String walDir = "mockFS://mockFSAuthority:9876/mockDir/wals/";
143    String baseDir = CommonFSUtils.getRootDir(UTIL.getConfiguration()).toString() + "/";
144    testArchiveStoreFilesDifferentFileSystems(walDir, baseDir, HFileArchiver::archiveStoreFiles);
145  }
146
147  @Test
148  public void testArchiveStoreFilesDifferentFileSystemsWallNullPlainRoot() throws Exception {
149    String baseDir = CommonFSUtils.getRootDir(UTIL.getConfiguration()).toString() + "/";
150    testArchiveStoreFilesDifferentFileSystems(null, baseDir, HFileArchiver::archiveStoreFiles);
151  }
152
153  @Test
154  public void testArchiveStoreFilesDifferentFileSystemsWallAndRootSame() throws Exception {
155    String baseDir = CommonFSUtils.getRootDir(UTIL.getConfiguration()).toString() + "/";
156    testArchiveStoreFilesDifferentFileSystems("/hbase/wals/", baseDir,
157      HFileArchiver::archiveStoreFiles);
158  }
159
160  @Test
161  public void testArchiveStoreFilesDifferentFileSystemsFileAlreadyArchived() throws Exception {
162    String baseDir = CommonFSUtils.getRootDir(UTIL.getConfiguration()).toString() + "/";
163    testArchiveStoreFilesDifferentFileSystems("/hbase/wals/", baseDir, true, false, false,
164      HFileArchiver::archiveStoreFiles);
165  }
166
167  @Test
168  public void testArchiveStoreFilesDifferentFileSystemsArchiveFileMatchCurrent() throws Exception {
169    String baseDir = CommonFSUtils.getRootDir(UTIL.getConfiguration()).toString() + "/";
170    testArchiveStoreFilesDifferentFileSystems("/hbase/wals/", baseDir, true, true, false,
171      HFileArchiver::archiveStoreFiles);
172  }
173
174  @Test(expected = IOException.class)
175  public void testArchiveStoreFilesDifferentFileSystemsArchiveFileMismatch() throws Exception {
176    String baseDir = CommonFSUtils.getRootDir(UTIL.getConfiguration()).toString() + "/";
177    testArchiveStoreFilesDifferentFileSystems("/hbase/wals/", baseDir, true, true, true,
178      HFileArchiver::archiveStoreFiles);
179  }
180
181  private void testArchiveStoreFilesDifferentFileSystems(String walDir, String expectedBase,
182    ArchivingFunction<Configuration, FileSystem, RegionInfo, Path, byte[],
183      Collection<HStoreFile>> archivingFunction)
184    throws IOException {
185    testArchiveStoreFilesDifferentFileSystems(walDir, expectedBase, false, true, false,
186      archivingFunction);
187  }
188
189  private void testArchiveStoreFilesDifferentFileSystems(String walDir, String expectedBase,
190    boolean archiveFileExists, boolean sourceFileExists, boolean archiveFileDifferentLength,
191    ArchivingFunction<Configuration, FileSystem, RegionInfo, Path, byte[],
192      Collection<HStoreFile>> archivingFunction)
193    throws IOException {
194    FileSystem mockedFileSystem = mock(FileSystem.class);
195    Configuration conf = new Configuration(UTIL.getConfiguration());
196    if (walDir != null) {
197      conf.set(CommonFSUtils.HBASE_WAL_DIR, walDir);
198    }
199    when(mockedFileSystem.getScheme()).thenReturn("mockFS");
200    when(mockedFileSystem.mkdirs(any())).thenReturn(true);
201    HashMap<Path, Boolean> existsTracker = new HashMap<>();
202    Path filePath = new Path("/mockDir/wals/mockFile");
203    String expectedDir = expectedBase
204      + "archive/data/default/mockTable/mocked-region-encoded-name/testfamily/mockFile";
205    existsTracker.put(new Path(expectedDir), archiveFileExists);
206    existsTracker.put(filePath, sourceFileExists);
207    when(mockedFileSystem.exists(any()))
208      .thenAnswer(invocation -> existsTracker.getOrDefault((Path) invocation.getArgument(0), true));
209    FileStatus mockedStatus = mock(FileStatus.class);
210    when(mockedStatus.getLen()).thenReturn(12L).thenReturn(archiveFileDifferentLength ? 34L : 12L);
211    when(mockedFileSystem.getFileStatus(any())).thenReturn(mockedStatus);
212    RegionInfo mockedRegion = mock(RegionInfo.class);
213    TableName tableName = TableName.valueOf("mockTable");
214    when(mockedRegion.getTable()).thenReturn(tableName);
215    when(mockedRegion.getEncodedName()).thenReturn("mocked-region-encoded-name");
216    Path tableDir = new Path("mockFS://mockDir/tabledir");
217    byte[] family = Bytes.toBytes("testfamily");
218    HStoreFile mockedFile = mock(HStoreFile.class);
219    List<HStoreFile> list = new ArrayList<>();
220    list.add(mockedFile);
221    when(mockedFile.getPath()).thenReturn(filePath);
222    when(mockedFileSystem.rename(any(), any())).thenReturn(true);
223    archivingFunction.apply(conf, mockedFileSystem, mockedRegion, tableDir, family, list);
224
225    if (sourceFileExists) {
226      ArgumentCaptor<Path> srcPath = ArgumentCaptor.forClass(Path.class);
227      ArgumentCaptor<Path> destPath = ArgumentCaptor.forClass(Path.class);
228      if (archiveFileExists) {
229        // Verify we renamed the archived file to sideline, and then renamed the source file.
230        verify(mockedFileSystem, times(2)).rename(srcPath.capture(), destPath.capture());
231        assertEquals(expectedDir, srcPath.getAllValues().get(0).toString());
232        assertEquals(filePath, srcPath.getAllValues().get(1));
233        assertEquals(expectedDir, destPath.getAllValues().get(1).toString());
234      } else {
235        // Verify we renamed the source file to the archived file.
236        verify(mockedFileSystem, times(1)).rename(srcPath.capture(), destPath.capture());
237        assertEquals(filePath, srcPath.getAllValues().get(0));
238        assertEquals(expectedDir, destPath.getAllValues().get(0).toString());
239      }
240    } else {
241      if (archiveFileExists) {
242        // Verify we did not rename. No source file with a present archive file should be a no-op.
243        verify(mockedFileSystem, never()).rename(any(), any());
244      } else {
245        fail("Unsupported test conditions: sourceFileExists and archiveFileExists both false.");
246      }
247    }
248  }
249
250  @FunctionalInterface
251  private interface ArchivingFunction<Configuration, FS, Region, Dir, Family, Files> {
252    void apply(Configuration config, FS fs, Region region, Dir dir, Family family, Files files)
253      throws IOException;
254  }
255
256  @Test
257  public void testArchiveRecoveredEditsWalDirNull() throws Exception {
258    testArchiveRecoveredEditsWalDirNullOrSame(null);
259  }
260
261  @Test
262  public void testArchiveRecoveredEditsWalDirSameFsStoreFiles() throws Exception {
263    testArchiveRecoveredEditsWalDirNullOrSame("/wal-dir");
264  }
265
266  private void testArchiveRecoveredEditsWalDirNullOrSame(String walDir) throws Exception {
267    String originalRootDir = UTIL.getConfiguration().get(HConstants.HBASE_DIR);
268    try {
269      String baseDir = "mockFS://mockFSAuthority:9876/hbase/";
270      UTIL.getConfiguration().set(HConstants.HBASE_DIR, baseDir);
271      testArchiveStoreFilesDifferentFileSystems(walDir, baseDir, (conf, fs, region, dir, family,
272        list) -> HFileArchiver.archiveRecoveredEdits(conf, fs, region, family, list));
273    } finally {
274      UTIL.getConfiguration().set(HConstants.HBASE_DIR, originalRootDir);
275    }
276  }
277
278  @Test(expected = IOException.class)
279  public void testArchiveRecoveredEditsWrongFS() throws Exception {
280    String baseDir = CommonFSUtils.getRootDir(UTIL.getConfiguration()).toString() + "/";
281    // Internally, testArchiveStoreFilesDifferentFileSystems will pass a "mockedFS"
282    // to HFileArchiver.archiveRecoveredEdits, but since wal-dir is supposedly on same FS
283    // as root dir it would lead to conflicting FSes and an IOException is expected.
284    testArchiveStoreFilesDifferentFileSystems("/wal-dir", baseDir, (conf, fs, region, dir, family,
285      list) -> HFileArchiver.archiveRecoveredEdits(conf, fs, region, family, list));
286  }
287
288  @Test
289  public void testArchiveRecoveredEditsWalDirDifferentFS() throws Exception {
290    String walDir = "mockFS://mockFSAuthority:9876/mockDir/wals/";
291    testArchiveStoreFilesDifferentFileSystems(walDir, walDir, (conf, fs, region, dir, family,
292      list) -> HFileArchiver.archiveRecoveredEdits(conf, fs, region, family, list));
293  }
294
295  @Test
296  public void testRemoveRegionDirOnArchive() throws Exception {
297    final TableName tableName = TableName.valueOf(name.getMethodName());
298    UTIL.createTable(tableName, TEST_FAM);
299
300    final Admin admin = UTIL.getAdmin();
301
302    // get the current store files for the region
303    List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(tableName);
304    // make sure we only have 1 region serving this table
305    assertEquals(1, servingRegions.size());
306    HRegion region = servingRegions.get(0);
307
308    // and load the table
309    UTIL.loadRegion(region, TEST_FAM);
310
311    // shutdown the table so we can manipulate the files
312    admin.disableTable(tableName);
313
314    FileSystem fs = UTIL.getTestFileSystem();
315
316    // now attempt to depose the region
317    Path rootDir = region.getRegionFileSystem().getTableDir().getParent();
318    Path regionDir = FSUtils.getRegionDirFromRootDir(rootDir, region.getRegionInfo());
319
320    HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
321
322    // check for the existence of the archive directory and some files in it
323    Path archiveDir = HFileArchiveTestingUtil.getRegionArchiveDir(UTIL.getConfiguration(), region);
324    assertTrue(fs.exists(archiveDir));
325
326    // check to make sure the store directory was copied
327    FileStatus[] stores = fs.listStatus(archiveDir, new PathFilter() {
328      @Override
329      public boolean accept(Path p) {
330        if (p.getName().contains(HConstants.RECOVERED_EDITS_DIR)) {
331          return false;
332        }
333        return true;
334      }
335    });
336    assertTrue(stores.length == 1);
337
338    // make sure we archived the store files
339    FileStatus[] storeFiles = fs.listStatus(stores[0].getPath());
340    assertTrue(storeFiles.length > 0);
341
342    // then ensure the region's directory isn't present
343    assertFalse(fs.exists(regionDir));
344
345    UTIL.deleteTable(tableName);
346  }
347
348  /**
349   * Test that the region directory is removed when we archive a region without store files, but
350   * still has hidden files.
351   */
352  @Test
353  public void testDeleteRegionWithNoStoreFiles() throws Exception {
354    final TableName tableName = TableName.valueOf(name.getMethodName());
355    UTIL.createTable(tableName, TEST_FAM);
356
357    // get the current store files for the region
358    List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(tableName);
359    // make sure we only have 1 region serving this table
360    assertEquals(1, servingRegions.size());
361    HRegion region = servingRegions.get(0);
362
363    FileSystem fs = region.getRegionFileSystem().getFileSystem();
364
365    // make sure there are some files in the regiondir
366    Path rootDir = CommonFSUtils.getRootDir(fs.getConf());
367    Path regionDir = FSUtils.getRegionDirFromRootDir(rootDir, region.getRegionInfo());
368    FileStatus[] regionFiles = CommonFSUtils.listStatus(fs, regionDir, null);
369    Assert.assertNotNull("No files in the region directory", regionFiles);
370    if (LOG.isDebugEnabled()) {
371      List<Path> files = new ArrayList<>();
372      for (FileStatus file : regionFiles) {
373        files.add(file.getPath());
374      }
375      LOG.debug("Current files:" + files);
376    }
377    // delete the visible folders so we just have hidden files/folders
378    final PathFilter dirFilter = new FSUtils.DirFilter(fs);
379    PathFilter nonHidden = new PathFilter() {
380      @Override
381      public boolean accept(Path file) {
382        return dirFilter.accept(file) && !file.getName().toString().startsWith(".");
383      }
384    };
385    FileStatus[] storeDirs = CommonFSUtils.listStatus(fs, regionDir, nonHidden);
386    for (FileStatus store : storeDirs) {
387      LOG.debug("Deleting store for test");
388      fs.delete(store.getPath(), true);
389    }
390
391    // then archive the region
392    HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
393
394    // and check to make sure the region directoy got deleted
395    assertFalse("Region directory (" + regionDir + "), still exists.", fs.exists(regionDir));
396
397    UTIL.deleteTable(tableName);
398  }
399
400  private List<HRegion> initTableForArchivingRegions(TableName tableName) throws IOException {
401    final byte[][] splitKeys =
402      new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("c"), Bytes.toBytes("d") };
403
404    UTIL.createTable(tableName, TEST_FAM, splitKeys);
405
406    // get the current store files for the regions
407    List<HRegion> regions = UTIL.getHBaseCluster().getRegions(tableName);
408    // make sure we have 4 regions serving this table
409    assertEquals(4, regions.size());
410
411    // and load the table
412    try (Table table = UTIL.getConnection().getTable(tableName)) {
413      UTIL.loadTable(table, TEST_FAM);
414    }
415
416    // disable the table so that we can manipulate the files
417    UTIL.getAdmin().disableTable(tableName);
418
419    return regions;
420  }
421
422  @Test
423  public void testArchiveRegions() throws Exception {
424    final TableName tableName = TableName.valueOf(name.getMethodName());
425    List<HRegion> regions = initTableForArchivingRegions(tableName);
426
427    FileSystem fs = UTIL.getTestFileSystem();
428
429    // now attempt to depose the regions
430    Path rootDir = CommonFSUtils.getRootDir(UTIL.getConfiguration());
431    Path tableDir = CommonFSUtils.getTableDir(rootDir, regions.get(0).getRegionInfo().getTable());
432    List<Path> regionDirList = regions.stream()
433      .map(region -> FSUtils.getRegionDirFromTableDir(tableDir, region.getRegionInfo()))
434      .collect(Collectors.toList());
435
436    HFileArchiver.archiveRegions(UTIL.getConfiguration(), fs, rootDir, tableDir, regionDirList);
437
438    // check for the existence of the archive directory and some files in it
439    for (HRegion region : regions) {
440      Path archiveDir =
441        HFileArchiveTestingUtil.getRegionArchiveDir(UTIL.getConfiguration(), region);
442      assertTrue(fs.exists(archiveDir));
443
444      // check to make sure the store directory was copied
445      FileStatus[] stores =
446        fs.listStatus(archiveDir, p -> !p.getName().contains(HConstants.RECOVERED_EDITS_DIR));
447      assertTrue(stores.length == 1);
448
449      // make sure we archived the store files
450      FileStatus[] storeFiles = fs.listStatus(stores[0].getPath());
451      assertTrue(storeFiles.length > 0);
452    }
453
454    // then ensure the region's directories aren't present
455    for (Path regionDir : regionDirList) {
456      assertFalse(fs.exists(regionDir));
457    }
458
459    UTIL.deleteTable(tableName);
460  }
461
462  @Test(expected = IOException.class)
463  public void testArchiveRegionsWhenPermissionDenied() throws Exception {
464    final TableName tableName = TableName.valueOf(name.getMethodName());
465    List<HRegion> regions = initTableForArchivingRegions(tableName);
466
467    // now attempt to depose the regions
468    Path rootDir = CommonFSUtils.getRootDir(UTIL.getConfiguration());
469    Path tableDir = CommonFSUtils.getTableDir(rootDir, regions.get(0).getRegionInfo().getTable());
470    List<Path> regionDirList = regions.stream()
471      .map(region -> FSUtils.getRegionDirFromTableDir(tableDir, region.getRegionInfo()))
472      .collect(Collectors.toList());
473
474    // To create a permission denied error, we do archive regions as a non-current user
475    UserGroupInformation ugi =
476      UserGroupInformation.createUserForTesting("foo1234", new String[] { "group1" });
477
478    try {
479      ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
480        FileSystem fs = UTIL.getTestFileSystem();
481        HFileArchiver.archiveRegions(UTIL.getConfiguration(), fs, rootDir, tableDir, regionDirList);
482        return null;
483      });
484    } catch (IOException e) {
485      assertTrue(e.getCause().getMessage().contains("Permission denied"));
486      throw e;
487    } finally {
488      UTIL.deleteTable(tableName);
489    }
490  }
491
492  @Test
493  public void testArchiveOnTableDelete() throws Exception {
494    final TableName tableName = TableName.valueOf(name.getMethodName());
495    UTIL.createTable(tableName, TEST_FAM);
496
497    List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(tableName);
498    // make sure we only have 1 region serving this table
499    assertEquals(1, servingRegions.size());
500    HRegion region = servingRegions.get(0);
501
502    // get the parent RS and monitor
503    HRegionServer hrs = UTIL.getRSForFirstRegionInTable(tableName);
504    FileSystem fs = hrs.getFileSystem();
505
506    // put some data on the region
507    LOG.debug("-------Loading table");
508    UTIL.loadRegion(region, TEST_FAM);
509
510    // get the hfiles in the region
511    List<HRegion> regions = hrs.getRegions(tableName);
512    assertEquals("More that 1 region for test table.", 1, regions.size());
513
514    region = regions.get(0);
515    // wait for all the compactions to complete
516    region.waitForFlushesAndCompactions();
517
518    // disable table to prevent new updates
519    UTIL.getAdmin().disableTable(tableName);
520    LOG.debug("Disabled table");
521
522    // remove all the files from the archive to get a fair comparison
523    clearArchiveDirectory();
524
525    // then get the current store files
526    byte[][] columns = region.getTableDescriptor().getColumnFamilyNames().toArray(new byte[0][]);
527    List<String> storeFiles = region.getStoreFileList(columns);
528
529    // then delete the table so the hfiles get archived
530    UTIL.deleteTable(tableName);
531    LOG.debug("Deleted table");
532
533    assertArchiveFiles(fs, storeFiles, 30000);
534  }
535
536  private void assertArchiveFiles(FileSystem fs, List<String> storeFiles, long timeout)
537    throws IOException {
538    long end = EnvironmentEdgeManager.currentTime() + timeout;
539    Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
540    List<String> archivedFiles = new ArrayList<>();
541
542    // We have to ensure that the DeleteTableHandler is finished. HBaseAdmin.deleteXXX() can return
543    // before all files
544    // are archived. We should fix HBASE-5487 and fix synchronous operations from admin.
545    while (EnvironmentEdgeManager.currentTime() < end) {
546      archivedFiles = getAllFileNames(fs, archiveDir);
547      if (archivedFiles.size() >= storeFiles.size()) {
548        break;
549      }
550    }
551
552    Collections.sort(storeFiles);
553    Collections.sort(archivedFiles);
554
555    LOG.debug("Store files:");
556    for (int i = 0; i < storeFiles.size(); i++) {
557      LOG.debug(i + " - " + storeFiles.get(i));
558    }
559    LOG.debug("Archive files:");
560    for (int i = 0; i < archivedFiles.size(); i++) {
561      LOG.debug(i + " - " + archivedFiles.get(i));
562    }
563
564    assertTrue("Archived files are missing some of the store files!",
565      archivedFiles.containsAll(storeFiles));
566  }
567
568  /**
569   * Test that the store files are archived when a column family is removed.
570   */
571  @Test
572  public void testArchiveOnTableFamilyDelete() throws Exception {
573    final TableName tableName = TableName.valueOf(name.getMethodName());
574    UTIL.createTable(tableName, new byte[][] { TEST_FAM, Bytes.toBytes("fam2") });
575
576    List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(tableName);
577    // make sure we only have 1 region serving this table
578    assertEquals(1, servingRegions.size());
579    HRegion region = servingRegions.get(0);
580
581    // get the parent RS and monitor
582    HRegionServer hrs = UTIL.getRSForFirstRegionInTable(tableName);
583    FileSystem fs = hrs.getFileSystem();
584
585    // put some data on the region
586    LOG.debug("-------Loading table");
587    UTIL.loadRegion(region, TEST_FAM);
588
589    // get the hfiles in the region
590    List<HRegion> regions = hrs.getRegions(tableName);
591    assertEquals("More that 1 region for test table.", 1, regions.size());
592
593    region = regions.get(0);
594    // wait for all the compactions to complete
595    region.waitForFlushesAndCompactions();
596
597    // disable table to prevent new updates
598    UTIL.getAdmin().disableTable(tableName);
599    LOG.debug("Disabled table");
600
601    // remove all the files from the archive to get a fair comparison
602    clearArchiveDirectory();
603
604    // then get the current store files
605    byte[][] columns = region.getTableDescriptor().getColumnFamilyNames().toArray(new byte[0][]);
606    List<String> storeFiles = region.getStoreFileList(columns);
607
608    // then delete the table so the hfiles get archived
609    UTIL.getAdmin().deleteColumnFamily(tableName, TEST_FAM);
610
611    assertArchiveFiles(fs, storeFiles, 30000);
612
613    UTIL.deleteTable(tableName);
614  }
615
616  /**
617   * Test HFileArchiver.resolveAndArchive() race condition HBASE-7643
618   */
619  @Test
620  public void testCleaningRace() throws Exception {
621    final long TEST_TIME = 20 * 1000;
622    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
623
624    Configuration conf = UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
625    Path rootDir = UTIL.getDataTestDirOnTestFS("testCleaningRace");
626    FileSystem fs = UTIL.getTestFileSystem();
627
628    Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
629    Path regionDir = new Path(
630      CommonFSUtils.getTableDir(new Path("./"), TableName.valueOf(name.getMethodName())), "abcdef");
631    Path familyDir = new Path(regionDir, "cf");
632
633    Path sourceRegionDir = new Path(rootDir, regionDir);
634    fs.mkdirs(sourceRegionDir);
635
636    Stoppable stoppable = new StoppableImplementation();
637
638    // The cleaner should be looping without long pauses to reproduce the race condition.
639    HFileCleaner cleaner = new HFileCleaner(1, stoppable, conf, fs, archiveDir, POOL);
640    try {
641      choreService.scheduleChore(cleaner);
642
643      // Keep creating/archiving new files while the cleaner is running in the other thread
644      long startTime = EnvironmentEdgeManager.currentTime();
645      for (long fid = 0; (EnvironmentEdgeManager.currentTime() - startTime) < TEST_TIME; ++fid) {
646        Path file = new Path(familyDir, String.valueOf(fid));
647        Path sourceFile = new Path(rootDir, file);
648        Path archiveFile = new Path(archiveDir, file);
649
650        fs.createNewFile(sourceFile);
651
652        try {
653          // Try to archive the file
654          HFileArchiver.archiveRegion(fs, rootDir, sourceRegionDir.getParent(), sourceRegionDir);
655
656          // The archiver succeded, the file is no longer in the original location
657          // but it's in the archive location.
658          LOG.debug("hfile=" + fid + " should be in the archive");
659          assertTrue(fs.exists(archiveFile));
660          assertFalse(fs.exists(sourceFile));
661        } catch (IOException e) {
662          // The archiver is unable to archive the file. Probably HBASE-7643 race condition.
663          // in this case, the file should not be archived, and we should have the file
664          // in the original location.
665          LOG.debug("hfile=" + fid + " should be in the source location");
666          assertFalse(fs.exists(archiveFile));
667          assertTrue(fs.exists(sourceFile));
668
669          // Avoid to have this file in the next run
670          fs.delete(sourceFile, false);
671        }
672      }
673    } finally {
674      stoppable.stop("test end");
675      cleaner.cancel(true);
676      choreService.shutdown();
677      fs.delete(rootDir, true);
678    }
679  }
680
681  @Test
682  public void testArchiveRegionTableAndRegionDirsNull() throws IOException {
683    Path rootDir = UTIL.getDataTestDirOnTestFS("testCleaningRace");
684    FileSystem fileSystem = UTIL.getTestFileSystem();
685    // Try to archive the file but with null regionDir, can't delete sourceFile
686    assertFalse(HFileArchiver.archiveRegion(fileSystem, rootDir, null, null));
687  }
688
689  @Test
690  public void testArchiveRegionWithTableDirNull() throws IOException {
691    Path regionDir = new Path(
692      CommonFSUtils.getTableDir(new Path("./"), TableName.valueOf(name.getMethodName())), "xyzabc");
693    Path familyDir = new Path(regionDir, "rd");
694    Path rootDir = UTIL.getDataTestDirOnTestFS("testCleaningRace");
695    Path file = new Path(familyDir, "1");
696    Path sourceFile = new Path(rootDir, file);
697    FileSystem fileSystem = UTIL.getTestFileSystem();
698    fileSystem.createNewFile(sourceFile);
699    Path sourceRegionDir = new Path(rootDir, regionDir);
700    fileSystem.mkdirs(sourceRegionDir);
701    // Try to archive the file
702    assertFalse(HFileArchiver.archiveRegion(fileSystem, rootDir, null, sourceRegionDir));
703    assertFalse(fileSystem.exists(sourceRegionDir));
704  }
705
706  @Test
707  public void testArchiveRegionWithRegionDirNull() throws IOException {
708    Path regionDir =
709      new Path(CommonFSUtils.getTableDir(new Path("./"), TableName.valueOf(name.getMethodName())),
710        "elgn4nf");
711    Path familyDir = new Path(regionDir, "rdar");
712    Path rootDir = UTIL.getDataTestDirOnTestFS("testCleaningRace");
713    Path file = new Path(familyDir, "2");
714    Path sourceFile = new Path(rootDir, file);
715    FileSystem fileSystem = UTIL.getTestFileSystem();
716    fileSystem.createNewFile(sourceFile);
717    Path sourceRegionDir = new Path(rootDir, regionDir);
718    fileSystem.mkdirs(sourceRegionDir);
719    // Try to archive the file but with null regionDir, can't delete sourceFile
720    assertFalse(
721      HFileArchiver.archiveRegion(fileSystem, rootDir, sourceRegionDir.getParent(), null));
722    assertTrue(fileSystem.exists(sourceRegionDir));
723    fileSystem.delete(sourceRegionDir, true);
724  }
725
726  private void clearArchiveDirectory() throws IOException {
727    UTIL.getTestFileSystem()
728      .delete(new Path(UTIL.getDefaultRootDirPath(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
729  }
730
731  /**
732   * Get the names of all the files below the given directory
733   * @param fs         the file system to inspect
734   * @param archiveDir the directory in which to look
735   * @return a list of all files in the directory and sub-directories
736   */
737  private List<String> getAllFileNames(final FileSystem fs, Path archiveDir) throws IOException {
738    FileStatus[] files = CommonFSUtils.listStatus(fs, archiveDir, new PathFilter() {
739      @Override
740      public boolean accept(Path p) {
741        if (p.getName().contains(HConstants.RECOVERED_EDITS_DIR)) {
742          return false;
743        }
744        return true;
745      }
746    });
747    return recurseOnFiles(fs, files, new ArrayList<>());
748  }
749
750  /** Recursively lookup all the file names under the file[] array **/
751  private List<String> recurseOnFiles(FileSystem fs, FileStatus[] files, List<String> fileNames)
752    throws IOException {
753    if (files == null || files.length == 0) return fileNames;
754
755    for (FileStatus file : files) {
756      if (file.isDirectory()) {
757        recurseOnFiles(fs, CommonFSUtils.listStatus(fs, file.getPath(), null), fileNames);
758      } else {
759        fileNames.add(file.getPath().getName());
760      }
761    }
762    return fileNames;
763  }
764}