Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.normalizer;
019
020import static java.lang.String.format;
021import static org.apache.hadoop.hbase.master.normalizer.RegionNormalizerWorker.CUMULATIVE_SIZE_LIMIT_MB_KEY;
022import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.DEFAULT_MERGE_MIN_REGION_AGE_DAYS;
023import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_ENABLED_KEY;
024import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_AGE_DAYS_KEY;
025import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_COUNT_KEY;
026import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_SIZE_MB_KEY;
027import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY;
028import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MIN_REGION_COUNT_KEY;
029import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.SPLIT_ENABLED_KEY;
030import static org.hamcrest.MatcherAssert.assertThat;
031import static org.hamcrest.Matchers.contains;
032import static org.hamcrest.Matchers.empty;
033import static org.hamcrest.Matchers.everyItem;
034import static org.hamcrest.Matchers.greaterThanOrEqualTo;
035import static org.hamcrest.Matchers.hasSize;
036import static org.hamcrest.Matchers.instanceOf;
037import static org.hamcrest.Matchers.not;
038import static org.junit.jupiter.api.Assertions.assertEquals;
039import static org.junit.jupiter.api.Assertions.assertFalse;
040import static org.junit.jupiter.api.Assertions.assertTrue;
041import static org.mockito.ArgumentMatchers.any;
042import static org.mockito.ArgumentMatchers.anyList;
043import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
044import static org.mockito.Mockito.spy;
045import static org.mockito.Mockito.times;
046import static org.mockito.Mockito.verify;
047import static org.mockito.Mockito.when;
048
049import java.time.Instant;
050import java.time.Period;
051import java.util.ArrayList;
052import java.util.Collections;
053import java.util.HashMap;
054import java.util.List;
055import java.util.Map;
056import org.apache.hadoop.conf.Configuration;
057import org.apache.hadoop.hbase.HBaseConfiguration;
058import org.apache.hadoop.hbase.RegionMetrics;
059import org.apache.hadoop.hbase.ServerName;
060import org.apache.hadoop.hbase.Size;
061import org.apache.hadoop.hbase.TableName;
062import org.apache.hadoop.hbase.client.RegionInfo;
063import org.apache.hadoop.hbase.client.RegionInfoBuilder;
064import org.apache.hadoop.hbase.client.TableDescriptor;
065import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
066import org.apache.hadoop.hbase.master.MasterServices;
067import org.apache.hadoop.hbase.master.RegionState;
068import org.apache.hadoop.hbase.testclassification.MasterTests;
069import org.apache.hadoop.hbase.testclassification.SmallTests;
070import org.apache.hadoop.hbase.util.Bytes;
071import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
072import org.junit.jupiter.api.BeforeEach;
073import org.junit.jupiter.api.Tag;
074import org.junit.jupiter.api.Test;
075import org.junit.jupiter.api.TestInfo;
076import org.mockito.Mockito;
077
078/**
079 * Tests logic of {@link SimpleRegionNormalizer}.
080 */
081@Tag(MasterTests.TAG)
082@Tag(SmallTests.TAG)
083public class TestSimpleRegionNormalizer {
084
085  private Configuration conf;
086  private SimpleRegionNormalizer normalizer;
087  private MasterServices masterServices;
088  private TableDescriptor tableDescriptor;
089  private TableName tableName;
090
091  @BeforeEach
092  public void before(TestInfo testInfo) {
093    conf = HBaseConfiguration.create();
094    tableName = TableName.valueOf(testInfo.getTestMethod().get().getName());
095    tableDescriptor = TableDescriptorBuilder.newBuilder(tableName).build();
096  }
097
098  @Test
099  public void testNoNormalizationForMetaTable() {
100    TableName testTable = TableName.META_TABLE_NAME;
101    TableDescriptor testMetaTd = TableDescriptorBuilder.newBuilder(testTable).build();
102    List<RegionInfo> RegionInfo = new ArrayList<>();
103    Map<byte[], Integer> regionSizes = new HashMap<>();
104
105    setupMocksForNormalizer(regionSizes, RegionInfo);
106    List<NormalizationPlan> plans = normalizer.computePlansForTable(testMetaTd);
107    assertThat(plans, empty());
108  }
109
110  @Test
111  public void testNoNormalizationIfTooFewRegions() {
112    final TableName tableName = this.tableName;
113    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 2);
114    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 10, 15);
115    setupMocksForNormalizer(regionSizes, regionInfos);
116
117    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
118    assertThat(plans, empty());
119  }
120
121  @Test
122  public void testNoNormalizationOnNormalizedCluster() {
123    final TableName tableName = this.tableName;
124    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 4);
125    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 10, 15, 8, 10);
126    setupMocksForNormalizer(regionSizes, regionInfos);
127
128    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
129    assertThat(plans, empty());
130  }
131
132  private void noNormalizationOnTransitioningRegions(final RegionState.State state) {
133    final TableName tableName = this.tableName;
134    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 3);
135    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 10, 1, 100);
136
137    setupMocksForNormalizer(regionSizes, regionInfos);
138    when(
139      masterServices.getAssignmentManager().getRegionStates().getRegionState(any(RegionInfo.class)))
140      .thenReturn(RegionState.createForTesting(null, state));
141    assertThat(normalizer.getMergeMinRegionCount(), greaterThanOrEqualTo(regionInfos.size()));
142
143    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
144    assertThat(format("Unexpected plans for RegionState %s", state), plans, empty());
145  }
146
147  @Test
148  public void testNoNormalizationOnMergingNewRegions() {
149    noNormalizationOnTransitioningRegions(RegionState.State.MERGING_NEW);
150  }
151
152  @Test
153  public void testNoNormalizationOnMergingRegions() {
154    noNormalizationOnTransitioningRegions(RegionState.State.MERGING);
155  }
156
157  @Test
158  public void testNoNormalizationOnMergedRegions() {
159    noNormalizationOnTransitioningRegions(RegionState.State.MERGED);
160  }
161
162  @Test
163  public void testNoNormalizationOnSplittingNewRegions() {
164    noNormalizationOnTransitioningRegions(RegionState.State.SPLITTING_NEW);
165  }
166
167  @Test
168  public void testNoNormalizationOnSplittingRegions() {
169    noNormalizationOnTransitioningRegions(RegionState.State.SPLITTING);
170  }
171
172  @Test
173  public void testNoNormalizationOnSplitRegions() {
174    noNormalizationOnTransitioningRegions(RegionState.State.SPLIT);
175  }
176
177  @Test
178  public void testMergeOfSmallRegions() {
179    final TableName tableName = this.tableName;
180    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
181    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 15, 5, 5, 15, 16);
182    setupMocksForNormalizer(regionSizes, regionInfos);
183
184    assertThat(normalizer.computePlansForTable(tableDescriptor),
185      contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(1), 5)
186        .addTarget(regionInfos.get(2), 5).build()));
187  }
188
189  // Test for situation illustrated in HBASE-14867
190  @Test
191  public void testMergeOfSecondSmallestRegions() {
192    final TableName tableName = this.tableName;
193    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 6);
194    final Map<byte[], Integer> regionSizes =
195      createRegionSizesMap(regionInfos, 1, 10000, 10000, 10000, 2700, 2700);
196    setupMocksForNormalizer(regionSizes, regionInfos);
197
198    assertThat(normalizer.computePlansForTable(tableDescriptor),
199      contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(4), 2700)
200        .addTarget(regionInfos.get(5), 2700).build()));
201  }
202
203  @Test
204  public void testMergeOfSmallNonAdjacentRegions() {
205    final TableName tableName = this.tableName;
206    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
207    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 15, 5, 16, 15, 5);
208    setupMocksForNormalizer(regionSizes, regionInfos);
209
210    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
211    assertThat(plans, empty());
212  }
213
214  @Test
215  public void testSplitOfLargeRegion() {
216    final TableName tableName = this.tableName;
217    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 4);
218    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 8, 6, 10, 30);
219    setupMocksForNormalizer(regionSizes, regionInfos);
220
221    assertThat(normalizer.computePlansForTable(tableDescriptor),
222      contains(new SplitNormalizationPlan(regionInfos.get(3), 30)));
223  }
224
225  @Test
226  public void testWithTargetRegionSize() throws Exception {
227    final TableName tableName = this.tableName;
228    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 6);
229    final Map<byte[], Integer> regionSizes =
230      createRegionSizesMap(regionInfos, 20, 40, 60, 80, 100, 120);
231    setupMocksForNormalizer(regionSizes, regionInfos);
232
233    // test when target region size is 20
234    when(tableDescriptor.getNormalizerTargetRegionSize()).thenReturn(20L);
235    assertThat(normalizer.computePlansForTable(tableDescriptor),
236      contains(new SplitNormalizationPlan(regionInfos.get(2), 60),
237        new SplitNormalizationPlan(regionInfos.get(3), 80),
238        new SplitNormalizationPlan(regionInfos.get(4), 100),
239        new SplitNormalizationPlan(regionInfos.get(5), 120)));
240
241    // test when target region size is 200
242    when(tableDescriptor.getNormalizerTargetRegionSize()).thenReturn(200L);
243    assertThat(normalizer.computePlansForTable(tableDescriptor),
244      contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 20)
245        .addTarget(regionInfos.get(1), 40).addTarget(regionInfos.get(2), 60)
246        .addTarget(regionInfos.get(3), 80).build()));
247  }
248
249  @Test
250  public void testSplitWithTargetRegionCount() throws Exception {
251    final TableName tableName = this.tableName;
252    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 4);
253    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 20, 40, 60, 80);
254    setupMocksForNormalizer(regionSizes, regionInfos);
255
256    // test when target region count is 8
257    when(tableDescriptor.getNormalizerTargetRegionCount()).thenReturn(8);
258    assertThat(normalizer.computePlansForTable(tableDescriptor),
259      contains(new SplitNormalizationPlan(regionInfos.get(2), 60),
260        new SplitNormalizationPlan(regionInfos.get(3), 80)));
261
262    // test when target region count is 3
263    when(tableDescriptor.getNormalizerTargetRegionCount()).thenReturn(3);
264    assertThat(normalizer.computePlansForTable(tableDescriptor),
265      contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 20)
266        .addTarget(regionInfos.get(1), 40).build()));
267  }
268
269  @Test
270  public void testHonorsSplitEnabled() {
271    conf.setBoolean(SPLIT_ENABLED_KEY, true);
272    final TableName tableName = this.tableName;
273    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
274    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 5, 5, 20, 5, 5);
275    setupMocksForNormalizer(regionSizes, regionInfos);
276    assertThat(normalizer.computePlansForTable(tableDescriptor),
277      contains(instanceOf(SplitNormalizationPlan.class)));
278
279    conf.setBoolean(SPLIT_ENABLED_KEY, false);
280    setupMocksForNormalizer(regionSizes, regionInfos);
281    assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
282  }
283
284  @Test
285  public void testHonorsSplitEnabledInTD() {
286    conf.setBoolean(SPLIT_ENABLED_KEY, true);
287    final TableName tableName = this.tableName;
288    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
289    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 5, 5, 20, 5, 5);
290    setupMocksForNormalizer(regionSizes, regionInfos);
291    assertThat(normalizer.computePlansForTable(tableDescriptor),
292      contains(instanceOf(SplitNormalizationPlan.class)));
293
294    // When hbase.normalizer.split.enabled is true in configuration, but false in table descriptor
295    when(tableDescriptor.getValue(SPLIT_ENABLED_KEY)).thenReturn("false");
296    assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
297
298    // When hbase.normalizer.split.enabled is false in configuration, but true in table descriptor
299    conf.setBoolean(SPLIT_ENABLED_KEY, false);
300    setupMocksForNormalizer(regionSizes, regionInfos);
301    when(tableDescriptor.getValue(SPLIT_ENABLED_KEY)).thenReturn("true");
302    assertThat(normalizer.computePlansForTable(tableDescriptor),
303      contains(instanceOf(SplitNormalizationPlan.class)));
304  }
305
306  @Test
307  public void testHonorsMergeEnabled() {
308    conf.setBoolean(MERGE_ENABLED_KEY, true);
309    final TableName tableName = this.tableName;
310    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
311    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 20, 5, 5, 20, 20);
312    setupMocksForNormalizer(regionSizes, regionInfos);
313    assertThat(normalizer.computePlansForTable(tableDescriptor),
314      contains(instanceOf(MergeNormalizationPlan.class)));
315
316    conf.setBoolean(MERGE_ENABLED_KEY, false);
317    setupMocksForNormalizer(regionSizes, regionInfos);
318    assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
319  }
320
321  @Test
322  public void testHonorsMergeEnabledInTD() {
323    conf.setBoolean(MERGE_ENABLED_KEY, true);
324    final TableName tableName = this.tableName;
325    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
326    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 20, 5, 5, 20, 20);
327    setupMocksForNormalizer(regionSizes, regionInfos);
328    assertThat(normalizer.computePlansForTable(tableDescriptor),
329      contains(instanceOf(MergeNormalizationPlan.class)));
330
331    // When hbase.normalizer.merge.enabled is true in configuration, but false in table descriptor
332    when(tableDescriptor.getValue(MERGE_ENABLED_KEY)).thenReturn("false");
333    assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
334
335    // When hbase.normalizer.merge.enabled is false in configuration, but true in table descriptor
336    conf.setBoolean(MERGE_ENABLED_KEY, false);
337    setupMocksForNormalizer(regionSizes, regionInfos);
338    when(tableDescriptor.getValue(MERGE_ENABLED_KEY)).thenReturn("true");
339    assertThat(normalizer.computePlansForTable(tableDescriptor),
340      contains(instanceOf(MergeNormalizationPlan.class)));
341  }
342
343  @Test
344  public void testHonorsMinimumRegionCount() {
345    honorsMinimumRegionCount(MERGE_MIN_REGION_COUNT_KEY);
346  }
347
348  /**
349   * Test the backward compatibility of the deprecated MIN_REGION_COUNT_KEY configuration.
350   */
351  @Test
352  public void testHonorsOldMinimumRegionCount() {
353    honorsMinimumRegionCount(MIN_REGION_COUNT_KEY);
354  }
355
356  private void honorsMinimumRegionCount(String confKey) {
357    conf.setInt(confKey, 1);
358    final TableName tableName = this.tableName;
359    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 3);
360    // create a table topology that results in both a merge plan and a split plan. Assert that the
361    // merge is only created when the when the number of table regions is above the region count
362    // threshold, and that the split plan is create in both cases.
363    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 1, 1, 10);
364    setupMocksForNormalizer(regionSizes, regionInfos);
365    assertEquals(1, normalizer.getMergeMinRegionCount());
366
367    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
368    assertThat(plans,
369      contains(new SplitNormalizationPlan(regionInfos.get(2), 10),
370        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 1)
371          .addTarget(regionInfos.get(1), 1).build()));
372
373    // have to call setupMocks again because we don't have dynamic config update on normalizer.
374    conf.setInt(confKey, 4);
375    setupMocksForNormalizer(regionSizes, regionInfos);
376    assertEquals(4, normalizer.getMergeMinRegionCount());
377    assertThat(normalizer.computePlansForTable(tableDescriptor),
378      contains(new SplitNormalizationPlan(regionInfos.get(2), 10)));
379  }
380
381  @Test
382  public void testHonorsMinimumRegionCountInTD() {
383    honorsOldMinimumRegionCountInTD(MERGE_MIN_REGION_COUNT_KEY);
384  }
385
386  /**
387   * Test the backward compatibility of the deprecated MIN_REGION_COUNT_KEY configuration in table
388   * descriptor.
389   */
390  @Test
391  public void testHonorsOldMinimumRegionCountInTD() {
392    honorsOldMinimumRegionCountInTD(MIN_REGION_COUNT_KEY);
393  }
394
395  private void honorsOldMinimumRegionCountInTD(String confKey) {
396    conf.setInt(confKey, 1);
397    final TableName tableName = this.tableName;
398    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 3);
399    // create a table topology that results in both a merge plan and a split plan. Assert that the
400    // merge is only created when the when the number of table regions is above the region count
401    // threshold, and that the split plan is create in both cases.
402    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 1, 1, 10);
403    setupMocksForNormalizer(regionSizes, regionInfos);
404    assertEquals(1, normalizer.getMergeMinRegionCount());
405
406    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
407    assertThat(plans,
408      contains(new SplitNormalizationPlan(regionInfos.get(2), 10),
409        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 1)
410          .addTarget(regionInfos.get(1), 1).build()));
411
412    when(tableDescriptor.getValue(confKey)).thenReturn("4");
413    assertThat(normalizer.computePlansForTable(tableDescriptor),
414      contains(new SplitNormalizationPlan(regionInfos.get(2), 10)));
415  }
416
417  @Test
418  public void testHonorsMergeMinRegionAge() {
419    conf.setInt(MERGE_MIN_REGION_AGE_DAYS_KEY, 7);
420    final TableName tableName = this.tableName;
421    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 4);
422    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 1, 1, 10, 10);
423    setupMocksForNormalizer(regionSizes, regionInfos);
424    assertEquals(Period.ofDays(7), normalizer.getMergeMinRegionAge());
425    assertThat(normalizer.computePlansForTable(tableDescriptor),
426      everyItem(not(instanceOf(MergeNormalizationPlan.class))));
427
428    // have to call setupMocks again because we don't have dynamic config update on normalizer.
429    conf.unset(MERGE_MIN_REGION_AGE_DAYS_KEY);
430    setupMocksForNormalizer(regionSizes, regionInfos);
431    assertEquals(Period.ofDays(DEFAULT_MERGE_MIN_REGION_AGE_DAYS),
432      normalizer.getMergeMinRegionAge());
433    final List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
434    assertThat(plans, not(empty()));
435    assertThat(plans, everyItem(instanceOf(MergeNormalizationPlan.class)));
436  }
437
438  @Test
439  public void testHonorsMergeMinRegionAgeInTD() {
440    conf.setInt(MERGE_MIN_REGION_AGE_DAYS_KEY, 7);
441    final TableName tableName = this.tableName;
442    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 4);
443    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 1, 1, 10, 10);
444    setupMocksForNormalizer(regionSizes, regionInfos);
445    assertEquals(Period.ofDays(7), normalizer.getMergeMinRegionAge());
446    assertThat(normalizer.computePlansForTable(tableDescriptor),
447      everyItem(not(instanceOf(MergeNormalizationPlan.class))));
448
449    conf.unset(MERGE_MIN_REGION_AGE_DAYS_KEY);
450    setupMocksForNormalizer(regionSizes, regionInfos);
451    when(tableDescriptor.getValue(MERGE_MIN_REGION_AGE_DAYS_KEY)).thenReturn("-1");
452    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
453    assertThat(plans, not(empty()));
454    assertThat(plans, everyItem(instanceOf(MergeNormalizationPlan.class)));
455
456    when(tableDescriptor.getValue(MERGE_MIN_REGION_AGE_DAYS_KEY)).thenReturn("5");
457    plans = normalizer.computePlansForTable(tableDescriptor);
458    assertThat(plans, empty());
459    assertThat(plans, everyItem(not(instanceOf(MergeNormalizationPlan.class))));
460  }
461
462  @Test
463  public void testHonorsMergeMinRegionSize() {
464    conf.setBoolean(SPLIT_ENABLED_KEY, false);
465    final TableName tableName = this.tableName;
466    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
467    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 1, 2, 0, 10, 10);
468    setupMocksForNormalizer(regionSizes, regionInfos);
469
470    assertFalse(normalizer.isSplitEnabled());
471    assertEquals(1, normalizer.getMergeMinRegionSizeMb());
472    assertThat(normalizer.computePlansForTable(tableDescriptor),
473      contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 1)
474        .addTarget(regionInfos.get(1), 2).build()));
475
476    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 3);
477    setupMocksForNormalizer(regionSizes, regionInfos);
478    assertEquals(3, normalizer.getMergeMinRegionSizeMb());
479    assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
480  }
481
482  @Test
483  public void testHonorsMergeMinRegionSizeInTD() {
484    conf.setBoolean(SPLIT_ENABLED_KEY, false);
485    final TableName tableName = this.tableName;
486    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
487    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 1, 2, 0, 10, 10);
488    setupMocksForNormalizer(regionSizes, regionInfos);
489
490    assertFalse(normalizer.isSplitEnabled());
491    assertEquals(1, normalizer.getMergeMinRegionSizeMb());
492    assertThat(normalizer.computePlansForTable(tableDescriptor),
493      contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 1)
494        .addTarget(regionInfos.get(1), 2).build()));
495
496    when(tableDescriptor.getValue(MERGE_MIN_REGION_SIZE_MB_KEY)).thenReturn("3");
497    assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
498  }
499
500  @Test
501  public void testHonorsMergeRequestMaxNumberOfRegionsCount() {
502    conf.setBoolean(SPLIT_ENABLED_KEY, false);
503    conf.setInt(MERGE_MIN_REGION_COUNT_KEY, 1);
504    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
505    conf.setInt(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, 3);
506    final TableName tableName = this.tableName;
507    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
508    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 0, 1, 0, 1, 0);
509    setupMocksForNormalizer(regionSizes, regionInfos);
510    assertEquals(3, normalizer.getMergeRequestMaxNumberOfRegionsCount());
511    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
512    assertThat(plans,
513      contains(
514        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
515          .addTarget(regionInfos.get(1), 1).addTarget(regionInfos.get(2), 0).build(),
516        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(3), 1)
517          .addTarget(regionInfos.get(4), 0).build()));
518  }
519
520  @Test
521  public void testHonorsMergeRequestMaxNumberOfRegionsCountDefault() {
522    conf.setBoolean(SPLIT_ENABLED_KEY, false);
523    conf.setInt(MERGE_MIN_REGION_COUNT_KEY, 1);
524    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
525    final TableName tableName = this.tableName;
526    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 3);
527    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 0, 0, 0);
528    setupMocksForNormalizer(regionSizes, regionInfos);
529    assertEquals(100, normalizer.getMergeRequestMaxNumberOfRegionsCount());
530    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
531    assertThat(plans, contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
532      .addTarget(regionInfos.get(1), 0).addTarget(regionInfos.get(2), 0).build()));
533  }
534
535  @Test
536  public void testMergeEmptyRegions0() {
537    conf.setBoolean(SPLIT_ENABLED_KEY, false);
538    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
539    final TableName tableName = this.tableName;
540    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 7);
541    final Map<byte[], Integer> regionSizes =
542      createRegionSizesMap(regionInfos, 0, 1, 10, 0, 9, 10, 0);
543    setupMocksForNormalizer(regionSizes, regionInfos);
544
545    assertFalse(normalizer.isSplitEnabled());
546    assertEquals(0, normalizer.getMergeMinRegionSizeMb());
547    assertThat(normalizer.computePlansForTable(tableDescriptor),
548      contains(
549        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
550          .addTarget(regionInfos.get(1), 1).build(),
551        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(2), 10)
552          .addTarget(regionInfos.get(3), 0).build(),
553        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(5), 10)
554          .addTarget(regionInfos.get(6), 0).build()));
555  }
556
557  @Test
558  public void testMergeEmptyRegions1() {
559    conf.setBoolean(SPLIT_ENABLED_KEY, false);
560    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
561    final TableName tableName = this.tableName;
562    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 8);
563    final Map<byte[], Integer> regionSizes =
564      createRegionSizesMap(regionInfos, 0, 1, 10, 0, 9, 0, 10, 0);
565    setupMocksForNormalizer(regionSizes, regionInfos);
566
567    assertFalse(normalizer.isSplitEnabled());
568    assertEquals(0, normalizer.getMergeMinRegionSizeMb());
569    assertThat(normalizer.computePlansForTable(tableDescriptor),
570      contains(
571        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
572          .addTarget(regionInfos.get(1), 1).build(),
573        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(2), 10)
574          .addTarget(regionInfos.get(3), 0).build(),
575        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(4), 9)
576          .addTarget(regionInfos.get(5), 0).build(),
577        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(6), 10)
578          .addTarget(regionInfos.get(7), 0).build()));
579  }
580
581  @Test
582  public void testMergeEmptyRegions2() {
583    conf.setBoolean(SPLIT_ENABLED_KEY, false);
584    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
585    final TableName tableName = this.tableName;
586    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 8);
587    final Map<byte[], Integer> regionSizes =
588      createRegionSizesMap(regionInfos, 0, 10, 1, 0, 9, 0, 10, 0);
589    setupMocksForNormalizer(regionSizes, regionInfos);
590
591    assertFalse(normalizer.isSplitEnabled());
592    assertEquals(0, normalizer.getMergeMinRegionSizeMb());
593    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
594    assertThat(plans,
595      contains(
596        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
597          .addTarget(regionInfos.get(1), 10).build(),
598        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(2), 1)
599          .addTarget(regionInfos.get(3), 0).build(),
600        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(4), 9)
601          .addTarget(regionInfos.get(5), 0).build(),
602        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(6), 10)
603          .addTarget(regionInfos.get(7), 0).build()));
604  }
605
606  @Test
607  public void testSplitAndMultiMerge() {
608    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
609    final TableName tableName = this.tableName;
610    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 8);
611    final Map<byte[], Integer> regionSizes =
612      createRegionSizesMap(regionInfos, 3, 1, 1, 30, 9, 3, 1, 0);
613    setupMocksForNormalizer(regionSizes, regionInfos);
614
615    assertTrue(normalizer.isMergeEnabled());
616    assertTrue(normalizer.isSplitEnabled());
617    assertEquals(0, normalizer.getMergeMinRegionSizeMb());
618    assertThat(normalizer.computePlansForTable(tableDescriptor),
619      contains(new SplitNormalizationPlan(regionInfos.get(3), 30),
620        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 3)
621          .addTarget(regionInfos.get(1), 1).addTarget(regionInfos.get(2), 1).build(),
622        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(5), 3)
623          .addTarget(regionInfos.get(6), 1).addTarget(regionInfos.get(7), 0).build()));
624  }
625
626  // This test is to make sure that normalizer is only going to merge adjacent regions.
627  @Test
628  public void testNormalizerCannotMergeNonAdjacentRegions() {
629    final TableName tableName = this.tableName;
630    // create 5 regions with sizes to trigger merge of small regions. region ranges are:
631    // [, "aa"), ["aa", "aa1"), ["aa1", "aa1!"), ["aa1!", "aa2"), ["aa2", )
632    // Region ["aa", "aa1") and ["aa1!", "aa2") are not adjacent, they are not supposed to
633    // merged.
634    final byte[][] keys = { null, Bytes.toBytes("aa"), Bytes.toBytes("aa1!"), Bytes.toBytes("aa1"),
635      Bytes.toBytes("aa2"), null, };
636    final List<RegionInfo> regionInfos = createRegionInfos(tableName, keys);
637    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 3, 1, 1, 3, 5);
638    setupMocksForNormalizer(regionSizes, regionInfos);
639
640    // Compute the plan, no merge plan returned as they are not adjacent.
641    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
642    assertThat(plans, empty());
643  }
644
645  @Test
646  public void testSizeLimitShufflesPlans() {
647    conf.setLong(CUMULATIVE_SIZE_LIMIT_MB_KEY, 10);
648    final TableName tableName = this.tableName;
649    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 4);
650    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 3, 3, 3, 3);
651    setupMocksForNormalizer(regionSizes, regionInfos);
652    when(tableDescriptor.getNormalizerTargetRegionSize()).thenReturn(1L);
653    normalizer = spy(normalizer);
654
655    assertTrue(normalizer.isSplitEnabled());
656    assertTrue(normalizer.isMergeEnabled());
657    List<NormalizationPlan> computedPlans = normalizer.computePlansForTable(tableDescriptor);
658    assertThat(computedPlans, hasSize(4));
659    verify(normalizer, times(1)).shuffleNormalizationPlans(anyList());
660  }
661
662  @SuppressWarnings("MockitoCast")
663  private void setupMocksForNormalizer(Map<byte[], Integer> regionSizes,
664    List<RegionInfo> regionInfoList) {
665    masterServices = Mockito.mock(MasterServices.class, RETURNS_DEEP_STUBS);
666    tableDescriptor = Mockito.mock(TableDescriptor.class, RETURNS_DEEP_STUBS);
667
668    // for simplicity all regions are assumed to be on one server; doesn't matter to us
669    ServerName sn = ServerName.valueOf("localhost", 0, 0L);
670    when(masterServices.getAssignmentManager().getRegionStates().getRegionsOfTable(any()))
671      .thenReturn(regionInfoList);
672    when(masterServices.getAssignmentManager().getRegionStates().getRegionServerOfRegion(any()))
673      .thenReturn(sn);
674    when(
675      masterServices.getAssignmentManager().getRegionStates().getRegionState(any(RegionInfo.class)))
676      .thenReturn(RegionState.createForTesting(null, RegionState.State.OPEN));
677
678    for (Map.Entry<byte[], Integer> region : regionSizes.entrySet()) {
679      RegionMetrics regionLoad = Mockito.mock(RegionMetrics.class);
680      when(regionLoad.getRegionName()).thenReturn(region.getKey());
681      when(regionLoad.getStoreFileSize())
682        .thenReturn(new Size(region.getValue(), Size.Unit.MEGABYTE));
683
684      // this is possibly broken with jdk9, unclear if false positive or not
685      // suppress it for now, fix it when we get to running tests on 9
686      // see: http://errorprone.info/bugpattern/MockitoCast
687      when((Object) masterServices.getServerManager().getLoad(sn).getRegionMetrics()
688        .get(region.getKey())).thenReturn(regionLoad);
689    }
690
691    when(masterServices.isSplitOrMergeEnabled(any())).thenReturn(true);
692    when(tableDescriptor.getTableName()).thenReturn(tableName);
693
694    normalizer = new SimpleRegionNormalizer();
695    normalizer.setConf(conf);
696    normalizer.setMasterServices(masterServices);
697  }
698
699  /**
700   * Create a list of {@link RegionInfo}s that represent a region chain of the specified length.
701   */
702  private static List<RegionInfo> createRegionInfos(final TableName tableName, final int length) {
703    if (length < 1) {
704      throw new IllegalStateException("length must be greater than or equal to 1.");
705    }
706
707    final byte[] startKey = Bytes.toBytes("aaaaa");
708    final byte[] endKey = Bytes.toBytes("zzzzz");
709    if (length == 1) {
710      return Collections.singletonList(createRegionInfo(tableName, startKey, endKey));
711    }
712
713    final byte[][] splitKeys = Bytes.split(startKey, endKey, length - 1);
714    final List<RegionInfo> ret = new ArrayList<>(length);
715    for (int i = 0; i < splitKeys.length - 1; i++) {
716      ret.add(createRegionInfo(tableName, splitKeys[i], splitKeys[i + 1]));
717    }
718    return ret;
719  }
720
721  private static RegionInfo createRegionInfo(final TableName tableName, final byte[] startKey,
722    final byte[] endKey) {
723    return RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(endKey)
724      .setRegionId(generateRegionId()).build();
725  }
726
727  private static long generateRegionId() {
728    return Instant.ofEpochMilli(EnvironmentEdgeManager.currentTime())
729      .minus(Period.ofDays(DEFAULT_MERGE_MIN_REGION_AGE_DAYS + 1)).toEpochMilli();
730  }
731
732  private static List<RegionInfo> createRegionInfos(final TableName tableName,
733    final byte[][] splitKeys) {
734    final List<RegionInfo> ret = new ArrayList<>(splitKeys.length);
735    for (int i = 0; i < splitKeys.length - 1; i++) {
736      ret.add(createRegionInfo(tableName, splitKeys[i], splitKeys[i + 1]));
737    }
738    return ret;
739  }
740
741  private static Map<byte[], Integer> createRegionSizesMap(final List<RegionInfo> regionInfos,
742    int... sizes) {
743    if (regionInfos.size() != sizes.length) {
744      throw new IllegalStateException("Parameter lengths must match.");
745    }
746
747    final Map<byte[], Integer> ret = new HashMap<>(regionInfos.size());
748    for (int i = 0; i < regionInfos.size(); i++) {
749      ret.put(regionInfos.get(i).getRegionName(), sizes[i]);
750    }
751    return ret;
752  }
753}