001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.compactions;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertThrows;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023
024import java.io.IOException;
025import java.util.List;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.DoNotRetryIOException;
028import org.apache.hadoop.hbase.HBaseTestingUtil;
029import org.apache.hadoop.hbase.HConstants;
030import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.Waiter;
033import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
034import org.apache.hadoop.hbase.client.Admin;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.Put;
037import org.apache.hadoop.hbase.client.Table;
038import org.apache.hadoop.hbase.client.TableDescriptor;
039import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
040import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
041import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
042import org.apache.hadoop.hbase.regionserver.HRegion;
043import org.apache.hadoop.hbase.regionserver.HRegionServer;
044import org.apache.hadoop.hbase.regionserver.HStore;
045import org.apache.hadoop.hbase.regionserver.HStoreFile;
046import org.apache.hadoop.hbase.testclassification.MediumTests;
047import org.apache.hadoop.hbase.testclassification.RegionServerTests;
048import org.apache.hadoop.hbase.util.Bytes;
049import org.apache.hadoop.hbase.util.EnvironmentEdge;
050import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
051import org.apache.hadoop.hbase.util.JVMClusterUtil;
052import org.apache.hadoop.hbase.util.TimeOffsetEnvironmentEdge;
053import org.junit.jupiter.api.AfterAll;
054import org.junit.jupiter.api.BeforeAll;
055import org.junit.jupiter.api.Tag;
056import org.junit.jupiter.api.Test;
057
058import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
059
060@Tag(RegionServerTests.TAG)
061@Tag(MediumTests.TAG)
062public class TestFIFOCompactionPolicy {
063
064  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
065
066  private final TableName tableName = TableName.valueOf(getClass().getSimpleName());
067
068  private final byte[] family = Bytes.toBytes("f");
069
070  private final byte[] qualifier = Bytes.toBytes("q");
071
072  private HStore getStoreWithName(TableName tableName) {
073    SingleProcessHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
074    List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
075    for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
076      HRegionServer hrs = rsts.get(i).getRegionServer();
077      for (HRegion region : hrs.getRegions(tableName)) {
078        return region.getStores().iterator().next();
079      }
080    }
081    return null;
082  }
083
084  private HStore prepareData() throws IOException {
085    Admin admin = TEST_UTIL.getAdmin();
086    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
087      .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
088        FIFOCompactionPolicy.class.getName())
089      .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName())
090      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1).build())
091      .build();
092    admin.createTable(desc);
093    Table table = TEST_UTIL.getConnection().getTable(tableName);
094    TimeOffsetEnvironmentEdge edge =
095      (TimeOffsetEnvironmentEdge) EnvironmentEdgeManager.getDelegate();
096    for (int i = 0; i < 10; i++) {
097      for (int j = 0; j < 10; j++) {
098        byte[] value = new byte[128 * 1024];
099        Bytes.random(value);
100        table.put(new Put(Bytes.toBytes(i * 10 + j)).addColumn(family, qualifier, value));
101      }
102      admin.flush(tableName);
103      edge.increment(1001);
104    }
105    return getStoreWithName(tableName);
106  }
107
108  @BeforeAll
109  public static void setEnvironmentEdge() throws Exception {
110    EnvironmentEdge ee = new TimeOffsetEnvironmentEdge();
111    EnvironmentEdgeManager.injectEdge(ee);
112    Configuration conf = TEST_UTIL.getConfiguration();
113    conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
114    // Expired store file deletion during compaction optimization interferes with the FIFO
115    // compaction policy. The race causes changes to in-flight-compaction files resulting in a
116    // non-deterministic number of files selected by compaction policy. Disables that optimization
117    // for this test run.
118    conf.setBoolean("hbase.store.delete.expired.storefile", false);
119    TEST_UTIL.startMiniCluster(1);
120  }
121
122  @AfterAll
123  public static void resetEnvironmentEdge() throws Exception {
124    TEST_UTIL.shutdownMiniCluster();
125    EnvironmentEdgeManager.reset();
126  }
127
128  @Test
129  public void testPurgeExpiredFiles() throws Exception {
130    HStore store = prepareData();
131    assertEquals(10, store.getStorefilesCount());
132    TEST_UTIL.getAdmin().majorCompact(tableName);
133    TEST_UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {
134
135      @Override
136      public boolean evaluate() throws Exception {
137        return store.getStorefilesCount() == 1;
138      }
139
140      @Override
141      public String explainFailure() throws Exception {
142        return "The store file count " + store.getStorefilesCount() + " is still greater than 1";
143      }
144    });
145  }
146
147  @Test
148  public void testSanityCheckTTL() throws IOException {
149    TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-TTL");
150    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
151      .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
152        FIFOCompactionPolicy.class.getName())
153      .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName())
154      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
155    DoNotRetryIOException e =
156      assertThrows(DoNotRetryIOException.class, () -> TEST_UTIL.getAdmin().createTable(desc));
157    assertTrue(e.getMessage().contains("Default TTL is not supported"));
158  }
159
160  @Test
161  public void testSanityCheckMinVersion() throws IOException {
162    TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-MinVersion");
163    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
164      .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
165        FIFOCompactionPolicy.class.getName())
166      .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName())
167      .setColumnFamily(
168        ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1).setMinVersions(1).build())
169      .build();
170    DoNotRetryIOException e =
171      assertThrows(DoNotRetryIOException.class, () -> TEST_UTIL.getAdmin().createTable(desc));
172    assertTrue(e.getMessage().contains("MIN_VERSION > 0 is not supported for FIFO compaction"));
173  }
174
175  @Test
176  public void testSanityCheckBlockingStoreFiles() throws IOException {
177    TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-BlockingStoreFiles");
178    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
179      .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
180        FIFOCompactionPolicy.class.getName())
181      .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName())
182      .setValue(HStore.BLOCKING_STOREFILES_KEY, "10")
183      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1).build())
184      .build();
185    DoNotRetryIOException e =
186      assertThrows(DoNotRetryIOException.class, () -> TEST_UTIL.getAdmin().createTable(desc));
187    assertTrue(e.getMessage().contains("Blocking file count 'hbase.hstore.blockingStoreFiles'"));
188    assertTrue(e.getMessage().contains("is below recommended minimum of 1000 for column family"));
189  }
190
191  /**
192   * Unit test for HBASE-21504
193   */
194  @Test
195  public void testFIFOCompactionPolicyExpiredEmptyHFiles() throws Exception {
196    TableName tableName = TableName.valueOf("testFIFOCompactionPolicyExpiredEmptyHFiles");
197    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
198      .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
199        FIFOCompactionPolicy.class.getName())
200      .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName())
201      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1).build())
202      .build();
203    Table table = TEST_UTIL.createTable(desc, null);
204    long ts = EnvironmentEdgeManager.currentTime() - 10 * 1000;
205    Put put =
206      new Put(Bytes.toBytes("row1")).addColumn(family, qualifier, ts, Bytes.toBytes("value0"));
207    table.put(put);
208    TEST_UTIL.getAdmin().flush(tableName); // HFile-0
209    put = new Put(Bytes.toBytes("row2")).addColumn(family, qualifier, ts, Bytes.toBytes("value1"));
210    table.put(put);
211    final int testWaitTimeoutMs = 20000;
212    TEST_UTIL.getAdmin().flush(tableName); // HFile-1
213
214    HStore store = Preconditions.checkNotNull(getStoreWithName(tableName));
215    assertEquals(2, store.getStorefilesCount());
216
217    TEST_UTIL.getAdmin().majorCompact(tableName);
218    TEST_UTIL.waitFor(testWaitTimeoutMs,
219      (Waiter.Predicate<Exception>) () -> store.getStorefilesCount() == 1);
220
221    assertEquals(1, store.getStorefilesCount());
222    HStoreFile sf = Preconditions.checkNotNull(store.getStorefiles().iterator().next());
223    assertEquals(0, sf.getReader().getEntries());
224
225    put = new Put(Bytes.toBytes("row3")).addColumn(family, qualifier, ts, Bytes.toBytes("value1"));
226    table.put(put);
227    TEST_UTIL.getAdmin().flush(tableName); // HFile-2
228    assertEquals(2, store.getStorefilesCount());
229
230    TEST_UTIL.getAdmin().majorCompact(tableName);
231    TEST_UTIL.waitFor(testWaitTimeoutMs,
232      (Waiter.Predicate<Exception>) () -> store.getStorefilesCount() == 1);
233
234    assertEquals(1, store.getStorefilesCount());
235    sf = Preconditions.checkNotNull(store.getStorefiles().iterator().next());
236    assertEquals(0, sf.getReader().getEntries());
237  }
238}