001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Collections;
026import java.util.List;
027import java.util.concurrent.atomic.AtomicBoolean;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.FileStatus;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.Put;
035import org.apache.hadoop.hbase.client.Table;
036import org.apache.hadoop.hbase.master.HMaster;
037import org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner;
038import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
039import org.apache.hadoop.hbase.testclassification.MediumTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.CommonFSUtils;
042import org.apache.hadoop.hbase.util.FSVisitor;
043import org.junit.jupiter.api.AfterAll;
044import org.junit.jupiter.api.BeforeAll;
045import org.junit.jupiter.api.Tag;
046import org.junit.jupiter.api.Test;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
051import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
052
053/**
054 * Test Case for HBASE-21387
055 */
056@Tag(MediumTests.TAG)
057public class TestSnapshotWhenChoreCleaning {
058
059  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
060  private static final Configuration CONF = TEST_UTIL.getConfiguration();
061  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotClientRetries.class);
062  private static final TableName TABLE_NAME = TableName.valueOf("testTable");
063  private static final int MAX_SPLIT_KEYS_NUM = 100;
064  private static final byte[] FAMILY = Bytes.toBytes("family");
065  private static final byte[] QUALIFIER = Bytes.toBytes("qualifier");
066  private static final byte[] VALUE = Bytes.toBytes("value");
067  private static Table TABLE;
068
069  @BeforeAll
070  public static void setUp() throws Exception {
071    // Set the hbase.snapshot.thread.pool.max to 1;
072    CONF.setInt("hbase.snapshot.thread.pool.max", 1);
073    // Enable snapshot
074    CONF.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
075    // Start MiniCluster.
076    TEST_UTIL.startMiniCluster(3);
077    // Create talbe
078    createTable();
079  }
080
081  private static byte[] integerToBytes(int i) {
082    return Bytes.toBytes(String.format("%06d", i));
083  }
084
085  private static void createTable() throws IOException {
086    byte[][] splitKeys = new byte[MAX_SPLIT_KEYS_NUM][];
087    for (int i = 0; i < splitKeys.length; i++) {
088      splitKeys[i] = integerToBytes(i);
089    }
090    TABLE = TEST_UTIL.createTable(TABLE_NAME, FAMILY, splitKeys);
091  }
092
093  @AfterAll
094  public static void tearDown() throws Exception {
095    TEST_UTIL.shutdownMiniCluster();
096  }
097
098  private static void loadDataAndFlush() throws IOException {
099    for (int i = 0; i < MAX_SPLIT_KEYS_NUM; i++) {
100      Put put =
101        new Put(integerToBytes(i)).addColumn(FAMILY, QUALIFIER, Bytes.add(VALUE, Bytes.toBytes(i)));
102      TABLE.put(put);
103    }
104    TEST_UTIL.flush(TABLE_NAME);
105  }
106
107  private static List<Path> listHFileNames(final FileSystem fs, final Path tableDir)
108    throws IOException {
109    final List<Path> hfiles = new ArrayList<>();
110    FSVisitor.visitTableStoreFiles(fs, tableDir, (region, family, hfileName) -> {
111      hfiles.add(new Path(new Path(new Path(tableDir, region), family), hfileName));
112    });
113    Collections.sort(hfiles);
114    return hfiles;
115  }
116
117  private static boolean isAnySnapshots(FileSystem fs) throws IOException {
118    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(CommonFSUtils.getRootDir(CONF));
119    FileStatus[] snapFiles = fs.listStatus(snapshotDir);
120    if (snapFiles.length == 0) {
121      return false;
122    }
123    Path firstPath = snapFiles[0].getPath();
124    LOG.info("firstPath in isAnySnapshots: " + firstPath);
125    if (snapFiles.length == 1 && firstPath.getName().equals(".tmp")) {
126      FileStatus[] tmpSnapFiles = fs.listStatus(firstPath);
127      return tmpSnapFiles != null && tmpSnapFiles.length > 0;
128    }
129    return true;
130  }
131
132  @Test
133  public void testSnapshotWhenSnapshotHFileCleanerRunning() throws Exception {
134    // Load data and flush to generate huge number of HFiles.
135    loadDataAndFlush();
136
137    SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner();
138    cleaner.init(ImmutableMap.of(HMaster.MASTER, TEST_UTIL.getHBaseCluster().getMaster()));
139    cleaner.setConf(CONF);
140
141    FileSystem fs = CommonFSUtils.getCurrentFileSystem(CONF);
142    List<Path> fileNames =
143      listHFileNames(fs, CommonFSUtils.getTableDir(CommonFSUtils.getRootDir(CONF), TABLE_NAME));
144    List<FileStatus> files = new ArrayList<>();
145    for (Path fileName : fileNames) {
146      files.add(fs.getFileStatus(fileName));
147    }
148
149    TEST_UTIL.getAdmin().snapshot("snapshotName_prev", TABLE_NAME);
150    assertEquals(0, Lists.newArrayList(cleaner.getDeletableFiles(files)).size());
151    TEST_UTIL.getAdmin().deleteSnapshot("snapshotName_prev");
152    cleaner.getFileCacheForTesting().triggerCacheRefreshForTesting();
153    assertEquals(100, Lists.newArrayList(cleaner.getDeletableFiles(files)).size());
154
155    Runnable snapshotRunnable = () -> {
156      try {
157        // The thread will be busy on taking snapshot;
158        for (int k = 0; k < 5; k++) {
159          TEST_UTIL.getAdmin().snapshot("snapshotName_" + k, TABLE_NAME);
160        }
161      } catch (Exception e) {
162        LOG.error("Snapshot failed: ", e);
163      }
164    };
165    final AtomicBoolean success = new AtomicBoolean(true);
166    Runnable cleanerRunnable = () -> {
167      try {
168        while (!isAnySnapshots(fs)) {
169          LOG.info("Not found any snapshot, sleep 100ms");
170          Thread.sleep(100);
171        }
172        for (int k = 0; k < 5; k++) {
173          cleaner.getFileCacheForTesting().triggerCacheRefreshForTesting();
174          Iterable<FileStatus> toDeleteFiles = cleaner.getDeletableFiles(files);
175          List<FileStatus> deletableFiles = Lists.newArrayList(toDeleteFiles);
176          LOG.info("Size of deletableFiles is: " + deletableFiles.size());
177          for (int i = 0; i < deletableFiles.size(); i++) {
178            LOG.debug("toDeleteFiles[{}] is: {}", i, deletableFiles.get(i));
179          }
180          if (deletableFiles.size() > 0) {
181            success.set(false);
182          }
183        }
184      } catch (Exception e) {
185        LOG.error("Chore cleaning failed: ", e);
186      }
187    };
188    Thread t1 = new Thread(snapshotRunnable);
189    t1.start();
190    Thread t2 = new Thread(cleanerRunnable);
191    t2.start();
192    t1.join();
193    t2.join();
194    assertTrue(success.get());
195  }
196}