001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import java.io.IOException;
021import org.apache.hadoop.hbase.HBaseClassTestRule;
022import org.apache.hadoop.hbase.HConstants;
023import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
024import org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory;
025import org.apache.hadoop.hbase.testclassification.RegionServerTests;
026import org.apache.hadoop.hbase.testclassification.SmallTests;
027import org.junit.ClassRule;
028import org.junit.Test;
029import org.junit.experimental.categories.Category;
030
031@Category({ RegionServerTests.class, SmallTests.class })
032public class TestDateTieredCompactionPolicy extends AbstractTestDateTieredCompactionPolicy {
033
034  @ClassRule
035  public static final HBaseClassTestRule CLASS_RULE =
036    HBaseClassTestRule.forClass(TestDateTieredCompactionPolicy.class);
037
038  @Override
039  protected void config() {
040    super.config();
041
042    // Set up policy
043    conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY,
044      "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
045    conf.setLong(CompactionConfiguration.DATE_TIERED_MAX_AGE_MILLIS_KEY, 100);
046    conf.setLong(CompactionConfiguration.DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 3);
047    conf.setLong(ExponentialCompactionWindowFactory.BASE_WINDOW_MILLIS_KEY, 6);
048    conf.setInt(ExponentialCompactionWindowFactory.WINDOWS_PER_TIER_KEY, 4);
049    conf.setBoolean(CompactionConfiguration.DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY,
050      false);
051
052    // Special settings for compaction policy per window
053    this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 2);
054    this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 12);
055    this.conf.setFloat(CompactionConfiguration.HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
056
057    conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 20);
058    conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 5);
059  }
060
061  /**
062   * Test for incoming window
063   * @throws IOException with error
064   */
065  @Test
066  public void incomingWindow() throws IOException {
067    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
068    long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
069    long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
070
071    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
072      new long[] { Long.MIN_VALUE, 12 }, false, true);
073  }
074
075  /**
076   * Not enough files in incoming window
077   * @throws IOException with error
078   */
079  @Test
080  public void NotIncomingWindow() throws IOException {
081    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
082    long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 };
083    long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11 };
084
085    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes),
086      new long[] { 20, 21, 22, 23, 24, 25 }, new long[] { Long.MIN_VALUE, 6 }, false, true);
087  }
088
089  /**
090   * Test for file on the upper bound of incoming window
091   * @throws IOException with error
092   */
093  @Test
094  public void OnUpperBoundOfIncomingWindow() throws IOException {
095    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
096    long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18 };
097    long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
098
099    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
100      new long[] { Long.MIN_VALUE, 12 }, false, true);
101  }
102
103  /**
104   * Test for file newer than incoming window
105   * @throws IOException with error
106   */
107  @Test
108  public void NewerThanIncomingWindow() throws IOException {
109    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
110    long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 19 };
111    long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
112
113    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
114      new long[] { Long.MIN_VALUE, 12 }, false, true);
115  }
116
117  /**
118   * If there is no T1 window, we don't build T2
119   * @throws IOException with error
120   */
121  @Test
122  public void NoT2() throws IOException {
123    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
124    long[] maxTimestamps = new long[] { 44, 60, 61, 97, 100, 193 };
125    long[] sizes = new long[] { 0, 20, 21, 22, 23, 1 };
126
127    compactEquals(194, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 22, 23 },
128      new long[] { Long.MIN_VALUE, 96 }, false, true);
129  }
130
131  @Test
132  public void T1() throws IOException {
133    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
134    long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
135    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 30, 31, 32, 2, 1 };
136
137    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31, 32 },
138      new long[] { Long.MIN_VALUE, 120 }, false, true);
139  }
140
141  /**
142   * Apply exploring logic on non-incoming window
143   * @throws IOException with error
144   */
145  @Test
146  public void RatioT0() throws IOException {
147    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
148    long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 };
149    long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 280, 23, 24, 1 };
150
151    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21, 22 },
152      new long[] { Long.MIN_VALUE }, false, true);
153  }
154
155  /**
156   * Also apply ratio-based logic on t2 window
157   * @throws IOException with error
158   */
159  @Test
160  public void RatioT2() throws IOException {
161    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
162    long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
163    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 350, 30, 31, 2, 1 };
164
165    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31 },
166      new long[] { Long.MIN_VALUE }, false, true);
167  }
168
169  /**
170   * The next compaction call after testTieredCompactionRatioT0 is compacted
171   * @throws IOException with error
172   */
173  @Test
174  public void RatioT0Next() throws IOException {
175    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
176    long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 8, 9, 10, 11, 12 };
177    long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 280, 23, 24, 1 };
178
179    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 23, 24 },
180      new long[] { Long.MIN_VALUE }, false, true);
181  }
182
183  /**
184   * Older than now(161) - maxAge(100)
185   * @throws IOException with error
186   */
187  @Test
188  public void olderThanMaxAge() throws IOException {
189    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
190    long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 };
191    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
192
193    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
194      new long[] { 40, 41, 42, 33, 30, 31 }, new long[] { Long.MIN_VALUE, 96 }, false, true);
195  }
196
197  /**
198   * Out-of-order data
199   * @throws IOException with error
200   */
201  @Test
202  public void outOfOrder() throws IOException {
203    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
204    long[] maxTimestamps = new long[] { 0, 13, 3, 10, 11, 1, 2, 12, 14, 15 };
205    long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 28, 23, 24, 1 };
206
207    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes),
208      new long[] { 31, 32, 33, 34, 22, 28, 23, 24, 1 }, new long[] { Long.MIN_VALUE, 12 }, false,
209      true);
210  }
211
212  /**
213   * Negative epoch time
214   * @throws IOException with error
215   */
216  @Test
217  public void negativeEpochtime() throws IOException {
218    long[] minTimestamps =
219      new long[] { -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000 };
220    long[] maxTimestamps = new long[] { -28, -11, -10, -9, -8, -7, -6, -5, -4, -3 };
221    long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 25, 23, 24, 1 };
222
223    compactEquals(1, sfCreate(minTimestamps, maxTimestamps, sizes),
224      new long[] { 31, 32, 33, 34, 22, 25, 23, 24, 1 }, new long[] { Long.MIN_VALUE, -24 }, false,
225      true);
226  }
227
228  /**
229   * Major compaction
230   * @throws IOException with error
231   */
232  @Test
233  public void majorCompation() throws IOException {
234    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
235    long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 };
236    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
237
238    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
239      new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 },
240      new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
241  }
242
243  /**
244   * Major Compaction to check min max timestamp falling in the same window and also to check
245   * boundary condition in which case binary sort gives insertion point as length of the array n
246   */
247  @Test
248  public void checkMinMaxTimestampSameBoundary() throws IOException {
249    long[] minTimestamps = new long[] { 0, 26, 50, 90, 98, 122, 145, 151, 158, 166 };
250    long[] maxTimestamps = new long[] { 12, 46, 70, 95, 100, 140, 148, 155, 162, 174 };
251    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 };
252
253    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
254      new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 },
255      new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
256  }
257
258  /**
259   * Major compaction with negative numbers
260   * @throws IOException with error
261   */
262  @Test
263  public void negativeForMajor() throws IOException {
264    long[] minTimestamps =
265      new long[] { -155, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100 };
266    long[] maxTimestamps = new long[] { -8, -7, -6, -5, -4, -3, -2, -1, 0, 6, 13 };
267    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
268
269    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes),
270      new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 },
271      new long[] { Long.MIN_VALUE, -144, -120, -96, -72, -48, -24, 0, 6, 12 }, true, true);
272  }
273}