001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import java.io.IOException;
021import org.apache.hadoop.hbase.HConstants;
022import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
023import org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory;
024import org.apache.hadoop.hbase.testclassification.RegionServerTests;
025import org.apache.hadoop.hbase.testclassification.SmallTests;
026import org.junit.jupiter.api.Tag;
027import org.junit.jupiter.api.Test;
028
029@Tag(RegionServerTests.TAG)
030@Tag(SmallTests.TAG)
031public class TestDateTieredCompactionPolicy extends AbstractTestDateTieredCompactionPolicy {
032
033  @Override
034  protected void config() {
035    super.config();
036
037    // Set up policy
038    conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY,
039      "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
040    conf.setLong(CompactionConfiguration.DATE_TIERED_MAX_AGE_MILLIS_KEY, 100);
041    conf.setLong(CompactionConfiguration.DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 3);
042    conf.setLong(ExponentialCompactionWindowFactory.BASE_WINDOW_MILLIS_KEY, 6);
043    conf.setInt(ExponentialCompactionWindowFactory.WINDOWS_PER_TIER_KEY, 4);
044    conf.setBoolean(CompactionConfiguration.DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY,
045      false);
046
047    // Special settings for compaction policy per window
048    this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 2);
049    this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 12);
050    this.conf.setFloat(CompactionConfiguration.HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
051
052    conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 20);
053    conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 5);
054  }
055
056  /**
057   * Test for incoming window
058   * @throws IOException with error
059   */
060  @Test
061  public void incomingWindow() throws IOException {
062    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
063    long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
064    long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
065
066    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
067      new long[] { Long.MIN_VALUE, 12 }, false, true);
068  }
069
070  /**
071   * Not enough files in incoming window
072   * @throws IOException with error
073   */
074  @Test
075  public void NotIncomingWindow() throws IOException {
076    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
077    long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 };
078    long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11 };
079
080    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes),
081      new long[] { 20, 21, 22, 23, 24, 25 }, new long[] { Long.MIN_VALUE, 6 }, false, true);
082  }
083
084  /**
085   * Test for file on the upper bound of incoming window
086   * @throws IOException with error
087   */
088  @Test
089  public void OnUpperBoundOfIncomingWindow() throws IOException {
090    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
091    long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18 };
092    long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
093
094    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
095      new long[] { Long.MIN_VALUE, 12 }, false, true);
096  }
097
098  /**
099   * Test for file newer than incoming window
100   * @throws IOException with error
101   */
102  @Test
103  public void NewerThanIncomingWindow() throws IOException {
104    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
105    long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 19 };
106    long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
107
108    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
109      new long[] { Long.MIN_VALUE, 12 }, false, true);
110  }
111
112  /**
113   * If there is no T1 window, we don't build T2
114   * @throws IOException with error
115   */
116  @Test
117  public void NoT2() throws IOException {
118    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
119    long[] maxTimestamps = new long[] { 44, 60, 61, 97, 100, 193 };
120    long[] sizes = new long[] { 0, 20, 21, 22, 23, 1 };
121
122    compactEquals(194, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 22, 23 },
123      new long[] { Long.MIN_VALUE, 96 }, false, true);
124  }
125
126  @Test
127  public void T1() throws IOException {
128    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
129    long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
130    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 30, 31, 32, 2, 1 };
131
132    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31, 32 },
133      new long[] { Long.MIN_VALUE, 120 }, false, true);
134  }
135
136  /**
137   * Apply exploring logic on non-incoming window
138   * @throws IOException with error
139   */
140  @Test
141  public void RatioT0() throws IOException {
142    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
143    long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 };
144    long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 280, 23, 24, 1 };
145
146    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21, 22 },
147      new long[] { Long.MIN_VALUE }, false, true);
148  }
149
150  /**
151   * Also apply ratio-based logic on t2 window
152   * @throws IOException with error
153   */
154  @Test
155  public void RatioT2() throws IOException {
156    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
157    long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
158    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 350, 30, 31, 2, 1 };
159
160    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31 },
161      new long[] { Long.MIN_VALUE }, false, true);
162  }
163
164  /**
165   * The next compaction call after testTieredCompactionRatioT0 is compacted
166   * @throws IOException with error
167   */
168  @Test
169  public void RatioT0Next() throws IOException {
170    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
171    long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 8, 9, 10, 11, 12 };
172    long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 280, 23, 24, 1 };
173
174    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 23, 24 },
175      new long[] { Long.MIN_VALUE }, false, true);
176  }
177
178  /**
179   * Older than now(161) - maxAge(100)
180   * @throws IOException with error
181   */
182  @Test
183  public void olderThanMaxAge() throws IOException {
184    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
185    long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 };
186    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
187
188    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
189      new long[] { 40, 41, 42, 33, 30, 31 }, new long[] { Long.MIN_VALUE, 96 }, false, true);
190  }
191
192  /**
193   * Out-of-order data
194   * @throws IOException with error
195   */
196  @Test
197  public void outOfOrder() throws IOException {
198    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
199    long[] maxTimestamps = new long[] { 0, 13, 3, 10, 11, 1, 2, 12, 14, 15 };
200    long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 28, 23, 24, 1 };
201
202    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes),
203      new long[] { 31, 32, 33, 34, 22, 28, 23, 24, 1 }, new long[] { Long.MIN_VALUE, 12 }, false,
204      true);
205  }
206
207  /**
208   * Negative epoch time
209   * @throws IOException with error
210   */
211  @Test
212  public void negativeEpochtime() throws IOException {
213    long[] minTimestamps =
214      new long[] { -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000 };
215    long[] maxTimestamps = new long[] { -28, -11, -10, -9, -8, -7, -6, -5, -4, -3 };
216    long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 25, 23, 24, 1 };
217
218    compactEquals(1, sfCreate(minTimestamps, maxTimestamps, sizes),
219      new long[] { 31, 32, 33, 34, 22, 25, 23, 24, 1 }, new long[] { Long.MIN_VALUE, -24 }, false,
220      true);
221  }
222
223  /**
224   * Major compaction
225   * @throws IOException with error
226   */
227  @Test
228  public void majorCompation() throws IOException {
229    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
230    long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 };
231    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
232
233    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
234      new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 },
235      new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
236  }
237
238  /**
239   * Major Compaction to check min max timestamp falling in the same window and also to check
240   * boundary condition in which case binary sort gives insertion point as length of the array
241   */
242  @Test
243  public void checkMinMaxTimestampSameBoundary() throws IOException {
244    long[] minTimestamps = new long[] { 0, 26, 50, 90, 98, 122, 145, 151, 158, 166 };
245    long[] maxTimestamps = new long[] { 12, 46, 70, 95, 100, 140, 148, 155, 162, 174 };
246    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 };
247
248    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
249      new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 },
250      new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
251  }
252
253  /**
254   * Major compaction with negative numbers
255   * @throws IOException with error
256   */
257  @Test
258  public void negativeForMajor() throws IOException {
259    long[] minTimestamps =
260      new long[] { -155, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100 };
261    long[] maxTimestamps = new long[] { -8, -7, -6, -5, -4, -3, -2, -1, 0, 6, 13 };
262    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
263
264    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes),
265      new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 },
266      new long[] { Long.MIN_VALUE, -144, -120, -96, -72, -48, -24, 0, 6, 12 }, true, true);
267  }
268}