Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.apache.hadoop.hbase.client.metrics.ScanMetrics.REGIONS_SCANNED_METRIC_NAME;
021import static org.apache.hadoop.hbase.client.metrics.ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME;
022import static org.junit.jupiter.api.Assertions.assertArrayEquals;
023import static org.junit.jupiter.api.Assertions.assertEquals;
024import static org.junit.jupiter.api.Assertions.assertNotNull;
025import static org.junit.jupiter.api.Assertions.assertNull;
026import static org.junit.jupiter.api.Assertions.fail;
027
028import java.io.IOException;
029import java.util.Arrays;
030import java.util.List;
031import java.util.Map;
032import java.util.stream.Collectors;
033import org.apache.hadoop.conf.Configuration;
034import org.apache.hadoop.fs.FileStatus;
035import org.apache.hadoop.fs.FileSystem;
036import org.apache.hadoop.fs.Path;
037import org.apache.hadoop.hbase.Cell;
038import org.apache.hadoop.hbase.CellScanner;
039import org.apache.hadoop.hbase.HBaseTestingUtil;
040import org.apache.hadoop.hbase.StartTestingClusterOption;
041import org.apache.hadoop.hbase.TableName;
042import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
043import org.apache.hadoop.hbase.client.metrics.ScanMetricsRegionInfo;
044import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner;
045import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
046import org.apache.hadoop.hbase.regionserver.HRegion;
047import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
048import org.apache.hadoop.hbase.regionserver.HRegionServer;
049import org.apache.hadoop.hbase.regionserver.StoreContext;
050import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
051import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
052import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils;
053import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
054import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
055import org.apache.hadoop.hbase.testclassification.ClientTests;
056import org.apache.hadoop.hbase.testclassification.LargeTests;
057import org.apache.hadoop.hbase.util.Bytes;
058import org.apache.hadoop.hbase.util.CommonFSUtils;
059import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
060import org.apache.hadoop.hbase.util.FSUtils;
061import org.apache.hadoop.hbase.util.HFileArchiveUtil;
062import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
063import org.junit.jupiter.api.AfterEach;
064import org.junit.jupiter.api.BeforeEach;
065import org.junit.jupiter.api.Tag;
066import org.junit.jupiter.api.Test;
067import org.junit.jupiter.api.TestInfo;
068import org.slf4j.Logger;
069import org.slf4j.LoggerFactory;
070
071@Tag(LargeTests.TAG)
072@Tag(ClientTests.TAG)
073public class TestTableSnapshotScanner {
074
075  private static final Logger LOG = LoggerFactory.getLogger(TestTableSnapshotScanner.class);
076  private final HBaseTestingUtil UTIL = new HBaseTestingUtil();
077  private static final int NUM_REGION_SERVERS = 2;
078  private static final byte[][] FAMILIES = { Bytes.toBytes("f1"), Bytes.toBytes("f2") };
079  public static byte[] bbb = Bytes.toBytes("bbb");
080  public static byte[] yyy = Bytes.toBytes("yyy");
081
082  private FileSystem fs;
083  private Path rootDir;
084  private boolean clusterUp;
085
086  private String methodName;
087
088  public static void blockUntilSplitFinished(HBaseTestingUtil util, TableName tableName,
089    int expectedRegionSize) throws Exception {
090    for (int i = 0; i < 100; i++) {
091      List<RegionInfo> hRegionInfoList = util.getAdmin().getRegions(tableName);
092      if (hRegionInfoList.size() >= expectedRegionSize) {
093        break;
094      }
095      Thread.sleep(1000);
096    }
097  }
098
099  @BeforeEach
100  public void setupCluster(TestInfo testInfo) throws Exception {
101    methodName = testInfo.getTestMethod().get().getName();
102    setupConf(UTIL.getConfiguration());
103    StartTestingClusterOption option =
104      StartTestingClusterOption.builder().numRegionServers(NUM_REGION_SERVERS)
105        .numDataNodes(NUM_REGION_SERVERS).createRootDir(true).build();
106    UTIL.startMiniCluster(option);
107    clusterUp = true;
108    rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
109    fs = rootDir.getFileSystem(UTIL.getConfiguration());
110  }
111
112  @AfterEach
113  public void tearDownCluster() throws Exception {
114    if (clusterUp) {
115      UTIL.shutdownMiniCluster();
116    }
117  }
118
119  protected void setupConf(Configuration conf) {
120    // Enable snapshot
121    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
122  }
123
124  public static void createTableAndSnapshot(HBaseTestingUtil util, TableName tableName,
125    String snapshotName, int numRegions) throws Exception {
126    try {
127      util.deleteTable(tableName);
128    } catch (Exception ex) {
129      // ignore
130    }
131
132    if (numRegions > 1) {
133      util.createTable(tableName, FAMILIES, 1, bbb, yyy, numRegions);
134    } else {
135      util.createTable(tableName, FAMILIES);
136    }
137    Admin admin = util.getAdmin();
138
139    // put some stuff in the table
140    Table table = util.getConnection().getTable(tableName);
141    util.loadTable(table, FAMILIES);
142
143    Path rootDir = CommonFSUtils.getRootDir(util.getConfiguration());
144    FileSystem fs = rootDir.getFileSystem(util.getConfiguration());
145
146    SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName, Arrays.asList(FAMILIES), null,
147      snapshotName, rootDir, fs, true);
148
149    // load different values
150    byte[] value = Bytes.toBytes("after_snapshot_value");
151    util.loadTable(table, FAMILIES, value);
152
153    // cause flush to create new files in the region
154    admin.flush(tableName);
155    table.close();
156  }
157
158  @Test
159  public void testNoDuplicateResultsWhenSplitting() throws Exception {
160    TableName tableName = TableName.valueOf("testNoDuplicateResultsWhenSplitting");
161    String snapshotName = "testSnapshotBug";
162    try {
163      if (UTIL.getAdmin().tableExists(tableName)) {
164        UTIL.deleteTable(tableName);
165      }
166
167      UTIL.createTable(tableName, FAMILIES);
168      Admin admin = UTIL.getAdmin();
169
170      // put some stuff in the table
171      Table table = UTIL.getConnection().getTable(tableName);
172      UTIL.loadTable(table, FAMILIES);
173
174      // split to 2 regions
175      admin.split(tableName, Bytes.toBytes("eee"));
176      blockUntilSplitFinished(UTIL, tableName, 2);
177
178      Path rootDir = CommonFSUtils.getRootDir(UTIL.getConfiguration());
179      FileSystem fs = rootDir.getFileSystem(UTIL.getConfiguration());
180
181      SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName, Arrays.asList(FAMILIES),
182        null, snapshotName, rootDir, fs, true);
183
184      // load different values
185      byte[] value = Bytes.toBytes("after_snapshot_value");
186      UTIL.loadTable(table, FAMILIES, value);
187
188      // cause flush to create new files in the region
189      admin.flush(tableName);
190      table.close();
191
192      Path restoreDir = UTIL.getDataTestDirOnTestFS(snapshotName);
193      Scan scan = new Scan().withStartRow(bbb).withStopRow(yyy); // limit the scan
194
195      TableSnapshotScanner scanner =
196        new TableSnapshotScanner(UTIL.getConfiguration(), restoreDir, snapshotName, scan);
197
198      verifyScanner(scanner, bbb, yyy);
199      scanner.close();
200    } catch (Exception e) {
201      e.printStackTrace();
202    } finally {
203      UTIL.getAdmin().deleteSnapshot(snapshotName);
204      UTIL.deleteTable(tableName);
205    }
206  }
207
208  @Test
209  public void testScanLimit() throws Exception {
210    final TableName tableName = TableName.valueOf(methodName);
211    final String snapshotName = tableName + "Snapshot";
212    TableSnapshotScanner scanner = null;
213    try {
214      createTableAndSnapshot(UTIL, tableName, snapshotName, 50);
215      Path restoreDir = UTIL.getDataTestDirOnTestFS(snapshotName);
216      Scan scan = new Scan().withStartRow(bbb).setLimit(100); // limit the scan
217
218      scanner = new TableSnapshotScanner(UTIL.getConfiguration(), restoreDir, snapshotName, scan);
219      int count = 0;
220      while (true) {
221        Result result = scanner.next();
222        if (result == null) {
223          break;
224        }
225        count++;
226      }
227      assertEquals(100, count);
228    } finally {
229      if (scanner != null) {
230        scanner.close();
231      }
232      UTIL.getAdmin().deleteSnapshot(snapshotName);
233      UTIL.deleteTable(tableName);
234    }
235  }
236
237  @Test
238  public void testWithSingleRegion() throws Exception {
239    testScanner(UTIL, "testWithSingleRegion", 1, false);
240  }
241
242  @Test
243  public void testWithMultiRegion() throws Exception {
244    testScanner(UTIL, "testWithMultiRegion", 10, false);
245  }
246
247  @Test
248  public void testWithOfflineHBaseMultiRegion() throws Exception {
249    testScanner(UTIL, "testWithMultiRegion", 20, true);
250  }
251
252  private ScanMetrics createTableSnapshotScannerAndGetScanMetrics(boolean enableScanMetrics,
253    boolean enableScanMetricsByRegion, byte[] endKey) throws Exception {
254    TableName tableName = TableName.valueOf(methodName + "_TABLE");
255    String snapshotName = methodName + "_SNAPSHOT";
256    try {
257      createTableAndSnapshot(UTIL, tableName, snapshotName, 50);
258      Path restoreDir = UTIL.getDataTestDirOnTestFS(snapshotName);
259      Scan scan = new Scan().withStartRow(bbb).withStopRow(endKey);
260      scan.setScanMetricsEnabled(enableScanMetrics);
261      scan.setEnableScanMetricsByRegion(enableScanMetricsByRegion);
262      Configuration conf = UTIL.getConfiguration();
263
264      TableSnapshotScanner snapshotScanner =
265        new TableSnapshotScanner(conf, restoreDir, snapshotName, scan);
266      verifyScanner(snapshotScanner, bbb, endKey);
267      return snapshotScanner.getScanMetrics();
268    } finally {
269      UTIL.getAdmin().deleteSnapshot(snapshotName);
270      UTIL.deleteTable(tableName);
271    }
272  }
273
274  @Test
275  public void testScanMetricsDisabled() throws Exception {
276    ScanMetrics scanMetrics = createTableSnapshotScannerAndGetScanMetrics(false, false, yyy);
277    assertNull(scanMetrics);
278  }
279
280  @Test
281  public void testScanMetricsWithScanMetricsByRegionDisabled() throws Exception {
282    ScanMetrics scanMetrics = createTableSnapshotScannerAndGetScanMetrics(true, false, yyy);
283    assertNotNull(scanMetrics);
284    int rowsScanned = 0;
285    for (byte[] row : HBaseTestingUtil.ROWS) {
286      if (Bytes.compareTo(row, bbb) >= 0 && Bytes.compareTo(row, yyy) < 0) {
287        rowsScanned++;
288      }
289    }
290    Map<String, Long> metricsMap = scanMetrics.getMetricsMap();
291    assertEquals(rowsScanned, (long) metricsMap.get(COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME));
292  }
293
294  @Test
295  public void testScanMetricsByRegionForSingleRegion() throws Exception {
296    // Scan single row with row key bbb
297    byte[] bbc = Bytes.toBytes("bbc");
298    ScanMetrics scanMetrics = createTableSnapshotScannerAndGetScanMetrics(true, true, bbc);
299    assertNotNull(scanMetrics);
300    Map<ScanMetricsRegionInfo, Map<String, Long>> scanMetricsByRegion =
301      scanMetrics.collectMetricsByRegion();
302    assertEquals(1, scanMetricsByRegion.size());
303    for (Map.Entry<ScanMetricsRegionInfo, Map<String, Long>> entry : scanMetricsByRegion
304      .entrySet()) {
305      ScanMetricsRegionInfo scanMetricsRegionInfo = entry.getKey();
306      Map<String, Long> metricsMap = entry.getValue();
307      assertNull(scanMetricsRegionInfo.getServerName());
308      assertNotNull(scanMetricsRegionInfo.getEncodedRegionName());
309      assertEquals(1, (long) metricsMap.get(REGIONS_SCANNED_METRIC_NAME));
310      assertEquals(1, (long) metricsMap.get(COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME));
311    }
312  }
313
314  @Test
315  public void testScanMetricsByRegionForMultiRegion() throws Exception {
316    ScanMetrics scanMetrics = createTableSnapshotScannerAndGetScanMetrics(true, true, yyy);
317    assertNotNull(scanMetrics);
318    Map<ScanMetricsRegionInfo, Map<String, Long>> scanMetricsByRegion =
319      scanMetrics.collectMetricsByRegion();
320    for (Map.Entry<ScanMetricsRegionInfo, Map<String, Long>> entry : scanMetricsByRegion
321      .entrySet()) {
322      ScanMetricsRegionInfo scanMetricsRegionInfo = entry.getKey();
323      Map<String, Long> metricsMap = entry.getValue();
324      assertNull(scanMetricsRegionInfo.getServerName());
325      assertNotNull(scanMetricsRegionInfo.getEncodedRegionName());
326      assertEquals(1, (long) metricsMap.get(REGIONS_SCANNED_METRIC_NAME));
327    }
328  }
329
330  @Test
331  public void testScannerWithRestoreScanner() throws Exception {
332    TableName tableName = TableName.valueOf("testScanner");
333    String snapshotName = "testScannerWithRestoreScanner";
334    try {
335      createTableAndSnapshot(UTIL, tableName, snapshotName, 50);
336      Path restoreDir = UTIL.getDataTestDirOnTestFS(snapshotName);
337      Scan scan = new Scan().withStartRow(bbb).withStopRow(yyy); // limit the scan
338
339      Configuration conf = UTIL.getConfiguration();
340      Path rootDir = CommonFSUtils.getRootDir(conf);
341
342      TableSnapshotScanner scanner0 =
343        new TableSnapshotScanner(conf, restoreDir, snapshotName, scan);
344      verifyScanner(scanner0, bbb, yyy);
345      scanner0.close();
346
347      // restore snapshot.
348      RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName);
349
350      // scan the snapshot without restoring snapshot
351      TableSnapshotScanner scanner =
352        new TableSnapshotScanner(conf, rootDir, restoreDir, snapshotName, scan, true);
353      verifyScanner(scanner, bbb, yyy);
354      scanner.close();
355
356      // check whether the snapshot has been deleted by the close of scanner.
357      scanner = new TableSnapshotScanner(conf, rootDir, restoreDir, snapshotName, scan, true);
358      verifyScanner(scanner, bbb, yyy);
359      scanner.close();
360
361      // restore snapshot again.
362      RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName);
363
364      // check whether the snapshot has been deleted by the close of scanner.
365      scanner = new TableSnapshotScanner(conf, rootDir, restoreDir, snapshotName, scan, true);
366      verifyScanner(scanner, bbb, yyy);
367      scanner.close();
368    } finally {
369      UTIL.getAdmin().deleteSnapshot(snapshotName);
370      UTIL.deleteTable(tableName);
371    }
372  }
373
374  @Test
375  public void testScannerWithRestoredMobSnapshot() throws Exception {
376    TableName tableName = TableName.valueOf(methodName);
377    String snapshotName = methodName + "Snapshot";
378    Path restoreDir = UTIL.getDataTestDirOnTestFS(snapshotName);
379    try {
380      MobSnapshotTestingUtils.createMobTable(UTIL, tableName, new byte[0][], 1, FAMILIES);
381      try (Table table = UTIL.getConnection().getTable(tableName)) {
382        UTIL.loadTable(table, FAMILIES);
383      }
384      UTIL.getAdmin().snapshot(snapshotName, tableName);
385
386      Configuration conf = UTIL.getConfiguration();
387      RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName);
388      try (TableSnapshotScanner scanner = new TableSnapshotScanner(conf, rootDir, restoreDir,
389        snapshotName, new Scan().withStartRow(Bytes.toBytes("zzzz")), true)) {
390        assertNull(scanner.next());
391      }
392    } finally {
393      fs.delete(restoreDir, true);
394      UTIL.getAdmin().deleteSnapshot(snapshotName);
395      UTIL.deleteTable(tableName);
396    }
397  }
398
399  private void testScanner(HBaseTestingUtil util, String snapshotName, int numRegions,
400    boolean shutdownCluster) throws Exception {
401    TableName tableName = TableName.valueOf("testScanner");
402    try {
403      createTableAndSnapshot(util, tableName, snapshotName, numRegions);
404
405      if (shutdownCluster) {
406        util.shutdownMiniHBaseCluster();
407        clusterUp = false;
408      }
409
410      Path restoreDir = util.getDataTestDirOnTestFS(snapshotName);
411      Scan scan = new Scan().withStartRow(bbb).withStopRow(yyy); // limit the scan
412
413      TableSnapshotScanner scanner =
414        new TableSnapshotScanner(UTIL.getConfiguration(), restoreDir, snapshotName, scan);
415
416      verifyScanner(scanner, bbb, yyy);
417      scanner.close();
418    } finally {
419      if (clusterUp) {
420        util.getAdmin().deleteSnapshot(snapshotName);
421        util.deleteTable(tableName);
422      }
423    }
424  }
425
426  private void verifyScanner(ResultScanner scanner, byte[] startRow, byte[] stopRow)
427    throws IOException, InterruptedException {
428
429    HBaseTestingUtil.SeenRowTracker rowTracker =
430      new HBaseTestingUtil.SeenRowTracker(startRow, stopRow);
431
432    while (true) {
433      Result result = scanner.next();
434      if (result == null) {
435        break;
436      }
437      verifyRow(result);
438      rowTracker.addRow(result.getRow());
439    }
440
441    // validate all rows are seen
442    rowTracker.validate();
443  }
444
445  private static void verifyRow(Result result) throws IOException {
446    byte[] row = result.getRow();
447    CellScanner scanner = result.cellScanner();
448    while (scanner.advance()) {
449      Cell cell = scanner.current();
450
451      // assert that all Cells in the Result have the same key
452      assertEquals(0, Bytes.compareTo(row, 0, row.length, cell.getRowArray(), cell.getRowOffset(),
453        cell.getRowLength()));
454    }
455
456    for (int j = 0; j < FAMILIES.length; j++) {
457      byte[] actual = result.getValue(FAMILIES[j], FAMILIES[j]);
458      assertArrayEquals(row, actual, "Row in snapshot does not match, expected:"
459        + Bytes.toString(row) + " ,actual:" + Bytes.toString(actual));
460    }
461  }
462
463  @Test
464  public void testMergeRegion() throws Exception {
465    TableName tableName = TableName.valueOf("testMergeRegion");
466    String snapshotName = tableName.getNameAsString() + "_snapshot";
467    Configuration conf = UTIL.getConfiguration();
468    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
469    long timeout = 20000; // 20s
470    try (Admin admin = UTIL.getAdmin()) {
471      List<String> serverList = admin.getRegionServers().stream().map(sn -> sn.getServerName())
472        .collect(Collectors.toList());
473      // create table with 3 regions
474      Table table = UTIL.createTable(tableName, FAMILIES, 1, bbb, yyy, 3);
475      List<RegionInfo> regions = admin.getRegions(tableName);
476      assertEquals(3, regions.size());
477      RegionInfo region0 = regions.get(0);
478      RegionInfo region1 = regions.get(1);
479      RegionInfo region2 = regions.get(2);
480      // put some data in the table
481      UTIL.loadTable(table, FAMILIES);
482      admin.flush(tableName);
483      // wait flush is finished
484      UTIL.waitFor(timeout, () -> {
485        try {
486          Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
487          for (RegionInfo region : regions) {
488            Path regionDir = new Path(tableDir, region.getEncodedName());
489            for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) {
490              for (FileStatus fs : fs.listStatus(familyDir)) {
491                if (!fs.getPath().getName().equals(".filelist")) {
492                  return true;
493                }
494              }
495              return false;
496            }
497          }
498          return true;
499        } catch (IOException e) {
500          LOG.warn("Failed check if flush is finished", e);
501          return false;
502        }
503      });
504      // merge 2 regions
505      admin.compactionSwitch(false, serverList);
506      admin.mergeRegionsAsync(region0.getEncodedNameAsBytes(), region1.getEncodedNameAsBytes(),
507        true);
508      UTIL.waitFor(timeout, () -> admin.getRegions(tableName).size() == 2);
509      List<RegionInfo> mergedRegions = admin.getRegions(tableName);
510      RegionInfo mergedRegion =
511        mergedRegions.get(0).getEncodedName().equals(region2.getEncodedName())
512          ? mergedRegions.get(1)
513          : mergedRegions.get(0);
514      // snapshot
515      admin.snapshot(snapshotName, tableName);
516      assertEquals(1, admin.listSnapshots().size());
517      // major compact
518      admin.compactionSwitch(true, serverList);
519      admin.majorCompactRegion(mergedRegion.getRegionName());
520      // wait until merged region has no reference
521      UTIL.waitFor(timeout, () -> {
522        try {
523          for (RegionServerThread regionServerThread : UTIL.getMiniHBaseCluster()
524            .getRegionServerThreads()) {
525            HRegionServer regionServer = regionServerThread.getRegionServer();
526            for (HRegion subRegion : regionServer.getRegions(tableName)) {
527              if (
528                subRegion.getRegionInfo().getEncodedName().equals(mergedRegion.getEncodedName())
529              ) {
530                regionServer.getCompactedHFilesDischarger().chore();
531              }
532            }
533          }
534          Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
535          HRegionFileSystem regionFs = HRegionFileSystem
536            .openRegionFromFileSystem(UTIL.getConfiguration(), fs, tableDir, mergedRegion, true);
537          boolean references = false;
538          Path regionDir = new Path(tableDir, mergedRegion.getEncodedName());
539          for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) {
540            StoreContext storeContext = StoreContext.getBuilder()
541              .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of(familyDir.getName()))
542              .withRegionFileSystem(regionFs).withFamilyStoreDirectoryPath(familyDir).build();
543            StoreFileTracker sft =
544              StoreFileTrackerFactory.create(UTIL.getConfiguration(), false, storeContext);
545            references = references || sft.hasReferences();
546            if (references) {
547              break;
548            }
549          }
550          return !references;
551        } catch (IOException e) {
552          LOG.warn("Failed check merged region has no reference", e);
553          return false;
554        }
555      });
556      // run catalog janitor to clean and wait for parent regions are archived
557      UTIL.getMiniHBaseCluster().getMaster().getCatalogJanitor().choreForTesting();
558      UTIL.waitFor(timeout, () -> {
559        try {
560          Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
561          for (FileStatus fileStatus : fs.listStatus(tableDir)) {
562            String name = fileStatus.getPath().getName();
563            if (name.equals(region0.getEncodedName()) || name.equals(region1.getEncodedName())) {
564              return false;
565            }
566          }
567          return true;
568        } catch (IOException e) {
569          LOG.warn("Check if parent regions are archived error", e);
570          return false;
571        }
572      });
573      // set file modify time and then run cleaner
574      long time = EnvironmentEdgeManager.currentTime() - TimeToLiveHFileCleaner.DEFAULT_TTL * 1000;
575      traverseAndSetFileTime(HFileArchiveUtil.getArchivePath(conf), time);
576      UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().triggerCleanerNow().get();
577      // scan snapshot
578      try (TableSnapshotScanner scanner =
579        new TableSnapshotScanner(conf, UTIL.getDataTestDirOnTestFS(snapshotName), snapshotName,
580          new Scan().withStartRow(bbb).withStopRow(yyy))) {
581        verifyScanner(scanner, bbb, yyy);
582      }
583    } catch (Exception e) {
584      LOG.error("scan snapshot error", e);
585      fail("Should not throw Exception: " + e.getMessage());
586    }
587  }
588
589  @Test
590  public void testDeleteTableWithMergedRegions() throws Exception {
591    final TableName tableName = TableName.valueOf(this.methodName);
592    String snapshotName = tableName.getNameAsString() + "_snapshot";
593    Configuration conf = UTIL.getConfiguration();
594    try (Admin admin = UTIL.getConnection().getAdmin()) {
595      // disable compaction
596      admin.compactionSwitch(false,
597        admin.getRegionServers().stream().map(s -> s.getServerName()).collect(Collectors.toList()));
598      // create table
599      Table table = UTIL.createTable(tableName, FAMILIES, 1, bbb, yyy, 3);
600      List<RegionInfo> regions = admin.getRegions(tableName);
601      assertEquals(3, regions.size());
602      // write some data
603      UTIL.loadTable(table, FAMILIES);
604      // merge region
605      admin.mergeRegionsAsync(new byte[][] { regions.get(0).getEncodedNameAsBytes(),
606        regions.get(1).getEncodedNameAsBytes() }, false).get();
607      regions = admin.getRegions(tableName);
608      assertEquals(2, regions.size());
609      // snapshot
610      admin.snapshot(snapshotName, tableName);
611      // verify snapshot
612      try (TableSnapshotScanner scanner =
613        new TableSnapshotScanner(conf, UTIL.getDataTestDirOnTestFS(snapshotName), snapshotName,
614          new Scan().withStartRow(bbb).withStopRow(yyy))) {
615        verifyScanner(scanner, bbb, yyy);
616      }
617      // drop table
618      admin.disableTable(tableName);
619      admin.deleteTable(tableName);
620      // verify snapshot
621      try (TableSnapshotScanner scanner =
622        new TableSnapshotScanner(conf, UTIL.getDataTestDirOnTestFS(snapshotName), snapshotName,
623          new Scan().withStartRow(bbb).withStopRow(yyy))) {
624        verifyScanner(scanner, bbb, yyy);
625      }
626    }
627  }
628
629  private void traverseAndSetFileTime(Path path, long time) throws IOException {
630    fs.setTimes(path, time, -1);
631    if (fs.isDirectory(path)) {
632      List<FileStatus> allPaths = Arrays.asList(fs.listStatus(path));
633      List<FileStatus> subDirs =
634        allPaths.stream().filter(FileStatus::isDirectory).collect(Collectors.toList());
635      List<FileStatus> files =
636        allPaths.stream().filter(FileStatus::isFile).collect(Collectors.toList());
637      for (FileStatus subDir : subDirs) {
638        traverseAndSetFileTime(subDir.getPath(), time);
639      }
640      for (FileStatus file : files) {
641        fs.setTimes(file.getPath(), time, -1);
642      }
643    }
644  }
645}