001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.compactions;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.util.List;
024import java.util.concurrent.ThreadLocalRandom;
025
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.DoNotRetryIOException;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.Waiter;
034import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
035import org.apache.hadoop.hbase.client.Admin;
036import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
037import org.apache.hadoop.hbase.client.Put;
038import org.apache.hadoop.hbase.client.Table;
039import org.apache.hadoop.hbase.client.TableDescriptor;
040import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
041import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
042import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
043import org.apache.hadoop.hbase.regionserver.HRegion;
044import org.apache.hadoop.hbase.regionserver.HRegionServer;
045import org.apache.hadoop.hbase.regionserver.HStore;
046import org.apache.hadoop.hbase.regionserver.HStoreFile;
047import org.apache.hadoop.hbase.testclassification.MediumTests;
048import org.apache.hadoop.hbase.testclassification.RegionServerTests;
049import org.apache.hadoop.hbase.util.Bytes;
050import org.apache.hadoop.hbase.util.EnvironmentEdge;
051import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
052import org.apache.hadoop.hbase.util.JVMClusterUtil;
053import org.apache.hadoop.hbase.util.TimeOffsetEnvironmentEdge;
054import org.junit.AfterClass;
055import org.junit.Assert;
056import org.junit.BeforeClass;
057import org.junit.ClassRule;
058import org.junit.Rule;
059import org.junit.Test;
060import org.junit.experimental.categories.Category;
061import org.junit.rules.ExpectedException;
062import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
063
064@Category({ RegionServerTests.class, MediumTests.class })
065public class TestFIFOCompactionPolicy {
066
067  @ClassRule
068  public static final HBaseClassTestRule CLASS_RULE =
069      HBaseClassTestRule.forClass(TestFIFOCompactionPolicy.class);
070
071  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
072
073  private final TableName tableName = TableName.valueOf(getClass().getSimpleName());
074
075  private final byte[] family = Bytes.toBytes("f");
076
077  private final byte[] qualifier = Bytes.toBytes("q");
078
079  @Rule
080  public ExpectedException error = ExpectedException.none();
081
082  private HStore getStoreWithName(TableName tableName) {
083    SingleProcessHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
084    List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
085    for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
086      HRegionServer hrs = rsts.get(i).getRegionServer();
087      for (HRegion region : hrs.getRegions(tableName)) {
088        return region.getStores().iterator().next();
089      }
090    }
091    return null;
092  }
093
094  private HStore prepareData() throws IOException {
095    Admin admin = TEST_UTIL.getAdmin();
096    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
097        .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
098          FIFOCompactionPolicy.class.getName())
099        .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
100          DisabledRegionSplitPolicy.class.getName())
101        .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1).build())
102        .build();
103    admin.createTable(desc);
104    Table table = TEST_UTIL.getConnection().getTable(tableName);
105    TimeOffsetEnvironmentEdge edge =
106        (TimeOffsetEnvironmentEdge) EnvironmentEdgeManager.getDelegate();
107    for (int i = 0; i < 10; i++) {
108      for (int j = 0; j < 10; j++) {
109        byte[] value = new byte[128 * 1024];
110        ThreadLocalRandom.current().nextBytes(value);
111        table.put(new Put(Bytes.toBytes(i * 10 + j)).addColumn(family, qualifier, value));
112      }
113      admin.flush(tableName);
114      edge.increment(1001);
115    }
116    return getStoreWithName(tableName);
117  }
118
119  @BeforeClass
120  public static void setEnvironmentEdge() throws Exception {
121    EnvironmentEdge ee = new TimeOffsetEnvironmentEdge();
122    EnvironmentEdgeManager.injectEdge(ee);
123    Configuration conf = TEST_UTIL.getConfiguration();
124    conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
125    // Expired store file deletion during compaction optimization interferes with the FIFO
126    // compaction policy. The race causes changes to in-flight-compaction files resulting in a
127    // non-deterministic number of files selected by compaction policy. Disables that optimization
128    // for this test run.
129    conf.setBoolean("hbase.store.delete.expired.storefile", false);
130    TEST_UTIL.startMiniCluster(1);
131  }
132
133  @AfterClass
134  public static void resetEnvironmentEdge() throws Exception {
135    TEST_UTIL.shutdownMiniCluster();
136    EnvironmentEdgeManager.reset();
137  }
138
139  @Test
140  public void testPurgeExpiredFiles() throws Exception {
141    HStore store = prepareData();
142    assertEquals(10, store.getStorefilesCount());
143    TEST_UTIL.getAdmin().majorCompact(tableName);
144    TEST_UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {
145
146      @Override
147      public boolean evaluate() throws Exception {
148        return store.getStorefilesCount() == 1;
149      }
150
151      @Override
152      public String explainFailure() throws Exception {
153        return "The store file count " + store.getStorefilesCount() + " is still greater than 1";
154      }
155    });
156  }
157
158  @Test
159  public void testSanityCheckTTL() throws IOException {
160    error.expect(DoNotRetryIOException.class);
161    error.expectMessage("Default TTL is not supported");
162    TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-TTL");
163    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
164        .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
165          FIFOCompactionPolicy.class.getName())
166        .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
167          DisabledRegionSplitPolicy.class.getName())
168        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
169    TEST_UTIL.getAdmin().createTable(desc);
170  }
171
172  @Test
173  public void testSanityCheckMinVersion() throws IOException {
174    error.expect(DoNotRetryIOException.class);
175    error.expectMessage("MIN_VERSION > 0 is not supported for FIFO compaction");
176    TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-MinVersion");
177    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
178        .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
179          FIFOCompactionPolicy.class.getName())
180        .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
181          DisabledRegionSplitPolicy.class.getName())
182        .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1)
183            .setMinVersions(1).build())
184        .build();
185    TEST_UTIL.getAdmin().createTable(desc);
186  }
187
188  @Test
189  public void testSanityCheckBlockingStoreFiles() throws IOException {
190    error.expect(DoNotRetryIOException.class);
191    error.expectMessage("Blocking file count 'hbase.hstore.blockingStoreFiles'");
192    error.expectMessage("is below recommended minimum of 1000 for column family");
193    TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-BlockingStoreFiles");
194    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
195        .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
196          FIFOCompactionPolicy.class.getName())
197        .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
198          DisabledRegionSplitPolicy.class.getName())
199        .setValue(HStore.BLOCKING_STOREFILES_KEY, "10")
200        .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1).build())
201        .build();
202    TEST_UTIL.getAdmin().createTable(desc);
203  }
204
205  /**
206   * Unit test for HBASE-21504
207   */
208  @Test
209  public void testFIFOCompactionPolicyExpiredEmptyHFiles() throws Exception {
210    TableName tableName = TableName.valueOf("testFIFOCompactionPolicyExpiredEmptyHFiles");
211    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
212        .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
213          FIFOCompactionPolicy.class.getName())
214        .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
215          DisabledRegionSplitPolicy.class.getName())
216        .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1).build())
217        .build();
218    Table table = TEST_UTIL.createTable(desc, null);
219    long ts = EnvironmentEdgeManager.currentTime() - 10 * 1000;
220    Put put =
221        new Put(Bytes.toBytes("row1")).addColumn(family, qualifier, ts, Bytes.toBytes("value0"));
222    table.put(put);
223    TEST_UTIL.getAdmin().flush(tableName); // HFile-0
224    put = new Put(Bytes.toBytes("row2")).addColumn(family, qualifier, ts, Bytes.toBytes("value1"));
225    table.put(put);
226    final int testWaitTimeoutMs = 20000;
227    TEST_UTIL.getAdmin().flush(tableName); // HFile-1
228
229    HStore store = Preconditions.checkNotNull(getStoreWithName(tableName));
230    Assert.assertEquals(2, store.getStorefilesCount());
231
232    TEST_UTIL.getAdmin().majorCompact(tableName);
233    TEST_UTIL.waitFor(testWaitTimeoutMs,
234        (Waiter.Predicate<Exception>) () -> store.getStorefilesCount() == 1);
235
236    Assert.assertEquals(1, store.getStorefilesCount());
237    HStoreFile sf = Preconditions.checkNotNull(store.getStorefiles().iterator().next());
238    Assert.assertEquals(0, sf.getReader().getEntries());
239
240    put = new Put(Bytes.toBytes("row3")).addColumn(family, qualifier, ts, Bytes.toBytes("value1"));
241    table.put(put);
242    TEST_UTIL.getAdmin().flush(tableName); // HFile-2
243    Assert.assertEquals(2, store.getStorefilesCount());
244
245    TEST_UTIL.getAdmin().majorCompact(tableName);
246    TEST_UTIL.waitFor(testWaitTimeoutMs,
247        (Waiter.Predicate<Exception>) () -> store.getStorefilesCount() == 1);
248
249    Assert.assertEquals(1, store.getStorefilesCount());
250    sf = Preconditions.checkNotNull(store.getStorefiles().iterator().next());
251    Assert.assertEquals(0, sf.getReader().getEntries());
252  }
253}