001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021
022import java.util.ArrayList;
023import java.util.List;
024import org.apache.hadoop.hbase.HBaseTestingUtil;
025import org.apache.hadoop.hbase.HConstants;
026import org.apache.hadoop.hbase.TableName;
027import org.apache.hadoop.hbase.client.Admin;
028import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
029import org.apache.hadoop.hbase.client.Put;
030import org.apache.hadoop.hbase.client.Scan;
031import org.apache.hadoop.hbase.client.Table;
032import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
033import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner;
034import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
035import org.apache.hadoop.hbase.testclassification.LargeTests;
036import org.apache.hadoop.hbase.testclassification.RegionServerTests;
037import org.apache.hadoop.hbase.util.Bytes;
038import org.apache.hadoop.hbase.util.JVMClusterUtil;
039import org.junit.jupiter.api.AfterAll;
040import org.junit.jupiter.api.AfterEach;
041import org.junit.jupiter.api.BeforeAll;
042import org.junit.jupiter.api.BeforeEach;
043import org.junit.jupiter.api.Tag;
044import org.junit.jupiter.api.Test;
045import org.slf4j.Logger;
046import org.slf4j.LoggerFactory;
047
048@Tag(LargeTests.TAG)
049@Tag(RegionServerTests.TAG)
050public class TestNotCleanupCompactedFileWhenRegionWarmup {
051  private static final Logger LOG =
052    LoggerFactory.getLogger(TestNotCleanupCompactedFileWhenRegionWarmup.class);
053
054  private static HBaseTestingUtil TEST_UTIL;
055  private static Admin admin;
056  private static Table table;
057
058  private static TableName TABLE_NAME = TableName.valueOf("TestCleanupCompactedFileAfterFailover");
059  private static byte[] ROW = Bytes.toBytes("row");
060  private static byte[] FAMILY = Bytes.toBytes("cf");
061  private static byte[] QUALIFIER = Bytes.toBytes("cq");
062  private static byte[] VALUE = Bytes.toBytes("value");
063
064  @BeforeAll
065  public static void beforeClass() throws Exception {
066    TEST_UTIL = new HBaseTestingUtil();
067    // Set the scanner lease to 20min, so the scanner can't be closed by RegionServer
068    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 1200000);
069    TEST_UTIL.getConfiguration().setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY,
070      100);
071    TEST_UTIL.getConfiguration().set("dfs.blocksize", "64000");
072    TEST_UTIL.getConfiguration().set("dfs.namenode.fs-limits.min-block-size", "1024");
073    TEST_UTIL.getConfiguration().set(TimeToLiveHFileCleaner.TTL_CONF_KEY, "0");
074    TEST_UTIL.startMiniCluster(1);
075    admin = TEST_UTIL.getAdmin();
076  }
077
078  @AfterAll
079  public static void afterClass() throws Exception {
080    TEST_UTIL.shutdownMiniCluster();
081  }
082
083  @BeforeEach
084  public void before() throws Exception {
085    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TABLE_NAME);
086    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY));
087    admin.createTable(builder.build());
088    TEST_UTIL.waitTableAvailable(TABLE_NAME);
089    table = TEST_UTIL.getConnection().getTable(TABLE_NAME);
090  }
091
092  @AfterEach
093  public void after() throws Exception {
094    admin.disableTable(TABLE_NAME);
095    admin.deleteTable(TABLE_NAME);
096  }
097
098  @Test
099  public void testRegionWarmup() throws Exception {
100    List<HRegion> regions = new ArrayList<>();
101    for (JVMClusterUtil.RegionServerThread rsThread : TEST_UTIL.getHBaseCluster()
102      .getLiveRegionServerThreads()) {
103      HRegionServer rs = rsThread.getRegionServer();
104      if (rs.getOnlineTables().contains(TABLE_NAME)) {
105        regions.addAll(rs.getRegions(TABLE_NAME));
106      }
107    }
108    assertEquals(1, regions.size(), "Table should only have one region");
109    HRegion region = regions.get(0);
110    HStore store = region.getStore(FAMILY);
111
112    writeDataAndFlush(3, region);
113    assertEquals(3, store.getStorefilesCount());
114
115    // Open a scanner and not close, then the storefile will be referenced
116    store.getScanner(new Scan(), null, 0);
117    region.compact(true);
118    assertEquals(1, store.getStorefilesCount());
119    // The compacted file should not be archived as there are references by user scanner
120    assertEquals(3, store.getStoreEngine().getStoreFileManager().getCompactedfiles().size());
121
122    HStore newStore = region.instantiateHStore(ColumnFamilyDescriptorBuilder.of(FAMILY), true);
123    // Should not archive the compacted storefiles when region warmup
124    assertEquals(4, newStore.getStorefilesCount());
125
126    newStore = region.instantiateHStore(ColumnFamilyDescriptorBuilder.of(FAMILY), false);
127    // Archived the compacted storefiles when region real open
128    assertEquals(1, newStore.getStorefilesCount());
129  }
130
131  private void writeDataAndFlush(int fileNum, HRegion region) throws Exception {
132    for (int i = 0; i < fileNum; i++) {
133      for (int j = 0; j < 100; j++) {
134        table.put(new Put(concat(ROW, j)).addColumn(FAMILY, QUALIFIER, concat(VALUE, j)));
135      }
136      region.flush(true);
137    }
138  }
139
140  private byte[] concat(byte[] base, int index) {
141    return Bytes.toBytes(Bytes.toString(base) + "-" + index);
142  }
143}