001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertNotNull;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.util.Collection;
026import org.apache.hadoop.hbase.HBaseTestingUtil;
027import org.apache.hadoop.hbase.TableName;
028import org.apache.hadoop.hbase.client.Admin;
029import org.apache.hadoop.hbase.client.Delete;
030import org.apache.hadoop.hbase.client.Get;
031import org.apache.hadoop.hbase.client.Put;
032import org.apache.hadoop.hbase.client.Result;
033import org.apache.hadoop.hbase.client.ResultScanner;
034import org.apache.hadoop.hbase.client.Scan;
035import org.apache.hadoop.hbase.client.Table;
036import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner;
037import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
038import org.apache.hadoop.hbase.testclassification.MediumTests;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.junit.jupiter.api.AfterAll;
041import org.junit.jupiter.api.BeforeAll;
042import org.junit.jupiter.api.Tag;
043import org.junit.jupiter.api.Test;
044
045@Tag(MediumTests.TAG)
046public class TestCleanupCompactedFileOnRegionClose {
047
048  private static HBaseTestingUtil util;
049
050  @BeforeAll
051  public static void beforeClass() throws Exception {
052    util = new HBaseTestingUtil();
053    util.getConfiguration().setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 100);
054    util.getConfiguration().set("dfs.blocksize", "64000");
055    util.getConfiguration().set("dfs.namenode.fs-limits.min-block-size", "1024");
056    util.getConfiguration().set(TimeToLiveHFileCleaner.TTL_CONF_KEY, "0");
057    util.startMiniCluster(2);
058  }
059
060  @AfterAll
061  public static void afterclass() throws Exception {
062    util.shutdownMiniCluster();
063  }
064
065  @Test
066  public void testCleanupOnClose() throws Exception {
067    TableName tableName = TableName.valueOf("testCleanupOnClose");
068    String familyName = "f";
069    byte[] familyNameBytes = Bytes.toBytes(familyName);
070    util.createTable(tableName, familyName);
071
072    Admin hBaseAdmin = util.getAdmin();
073    Table table = util.getConnection().getTable(tableName);
074
075    HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
076    Region region = rs.getRegions(tableName).get(0);
077
078    int refSFCount = 4;
079    for (int i = 0; i < refSFCount; i++) {
080      for (int j = 0; j < refSFCount; j++) {
081        Put put = new Put(Bytes.toBytes(j));
082        put.addColumn(familyNameBytes, Bytes.toBytes(i), Bytes.toBytes(j));
083        table.put(put);
084      }
085      util.flush(tableName);
086    }
087    assertEquals(refSFCount, region.getStoreFileList(new byte[][] { familyNameBytes }).size());
088
089    // add a delete, to test wether we end up with an inconsistency post region close
090    Delete delete = new Delete(Bytes.toBytes(refSFCount - 1));
091    table.delete(delete);
092    util.flush(tableName);
093    assertFalse(table.exists(new Get(Bytes.toBytes(refSFCount - 1))));
094
095    // Create a scanner and keep it open to add references to StoreFileReaders
096    Scan scan = new Scan();
097    scan.withStopRow(Bytes.toBytes(refSFCount - 2));
098    scan.setCaching(1);
099    ResultScanner scanner = table.getScanner(scan);
100    Result res = scanner.next();
101    assertNotNull(res);
102    assertEquals(refSFCount, res.getFamilyMap(familyNameBytes).size());
103
104    // Verify the references
105    int count = 0;
106    for (HStoreFile sf : (Collection<HStoreFile>) region.getStore(familyNameBytes)
107      .getStorefiles()) {
108      synchronized (sf) {
109        if (count < refSFCount) {
110          assertTrue(sf.isReferencedInReads());
111        } else {
112          assertFalse(sf.isReferencedInReads());
113        }
114      }
115      count++;
116    }
117
118    // Major compact to produce compacted storefiles that need to be cleaned up
119    util.compact(tableName, true);
120    assertEquals(1, region.getStoreFileList(new byte[][] { familyNameBytes }).size());
121    assertEquals(refSFCount + 1, ((HStore) region.getStore(familyNameBytes)).getStoreEngine()
122      .getStoreFileManager().getCompactedfiles().size());
123
124    // close then open the region to determine wether compacted storefiles get cleaned up on close
125    hBaseAdmin.unassign(region.getRegionInfo().getRegionName(), false);
126    hBaseAdmin.assign(region.getRegionInfo().getRegionName());
127    util.waitUntilNoRegionsInTransition(10000);
128
129    assertFalse(table.exists(new Get(Bytes.toBytes(refSFCount - 1))),
130      "Deleted row should not exist");
131
132    rs = util.getRSForFirstRegionInTable(tableName);
133    region = rs.getRegions(tableName).get(0);
134    assertEquals(1, region.getStoreFileList(new byte[][] { familyNameBytes }).size());
135    assertEquals(0, ((HStore) region.getStore(familyNameBytes)).getStoreEngine()
136      .getStoreFileManager().getCompactedfiles().size());
137  }
138}