Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.normalizer;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.mockito.Matchers.any;
023import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
024import static org.mockito.Mockito.when;
025
026import java.util.ArrayList;
027import java.util.HashMap;
028import java.util.List;
029import java.util.Map;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseIOException;
032import org.apache.hadoop.hbase.RegionMetrics;
033import org.apache.hadoop.hbase.ServerName;
034import org.apache.hadoop.hbase.Size;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.client.RegionInfoBuilder;
038import org.apache.hadoop.hbase.master.MasterRpcServices;
039import org.apache.hadoop.hbase.master.MasterServices;
040import org.apache.hadoop.hbase.testclassification.MasterTests;
041import org.apache.hadoop.hbase.testclassification.SmallTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.junit.Assert;
044import org.junit.BeforeClass;
045import org.junit.ClassRule;
046import org.junit.Rule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049import org.junit.rules.TestName;
050import org.mockito.Mockito;
051import org.slf4j.Logger;
052import org.slf4j.LoggerFactory;
053
054import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
055
056import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledResponse;
057
058/**
059 * Tests logic of {@link SimpleRegionNormalizer}.
060 */
061@Category({MasterTests.class, SmallTests.class})
062public class TestSimpleRegionNormalizer {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066      HBaseClassTestRule.forClass(TestSimpleRegionNormalizer.class);
067
068  private static final Logger LOG = LoggerFactory.getLogger(TestSimpleRegionNormalizer.class);
069
070  private static RegionNormalizer normalizer;
071
072  // mocks
073  private static MasterServices masterServices;
074  private static MasterRpcServices masterRpcServices;
075
076  @Rule
077  public TestName name = new TestName();
078
079  @BeforeClass
080  public static void beforeAllTests() throws Exception {
081    normalizer = new SimpleRegionNormalizer();
082  }
083
084  @Test
085  public void testNoNormalizationForMetaTable() throws HBaseIOException {
086    TableName testTable = TableName.META_TABLE_NAME;
087    List<RegionInfo> RegionInfo = new ArrayList<>();
088    Map<byte[], Integer> regionSizes = new HashMap<>();
089
090    setupMocksForNormalizer(regionSizes, RegionInfo);
091    List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable);
092    assertTrue(plans == null);
093  }
094
095  @Test
096  public void testNoNormalizationIfTooFewRegions() throws HBaseIOException {
097    final TableName tableName = TableName.valueOf(name.getMethodName());
098    List<RegionInfo> RegionInfo = new ArrayList<>();
099    Map<byte[], Integer> regionSizes = new HashMap<>();
100    RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
101        .setStartKey(Bytes.toBytes("aaa"))
102        .setEndKey(Bytes.toBytes("bbb"))
103        .build();
104    RegionInfo.add(hri1);
105    regionSizes.put(hri1.getRegionName(), 10);
106
107    RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
108        .setStartKey(Bytes.toBytes("bbb"))
109        .setEndKey(Bytes.toBytes("ccc"))
110        .build();
111    RegionInfo.add(hri2);
112    regionSizes.put(hri2.getRegionName(), 15);
113
114    setupMocksForNormalizer(regionSizes, RegionInfo);
115    List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
116    assertTrue(plans == null);
117  }
118
119  @Test
120  public void testNoNormalizationOnNormalizedCluster() throws HBaseIOException {
121    final TableName tableName = TableName.valueOf(name.getMethodName());
122    List<RegionInfo> RegionInfo = new ArrayList<>();
123    Map<byte[], Integer> regionSizes = new HashMap<>();
124
125    RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
126        .setStartKey(Bytes.toBytes("aaa"))
127        .setEndKey(Bytes.toBytes("bbb"))
128        .build();
129    RegionInfo.add(hri1);
130    regionSizes.put(hri1.getRegionName(), 10);
131
132    RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
133        .setStartKey(Bytes.toBytes("bbb"))
134        .setEndKey(Bytes.toBytes("ccc"))
135        .build();
136    RegionInfo.add(hri2);
137    regionSizes.put(hri2.getRegionName(), 15);
138
139    RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName)
140        .setStartKey(Bytes.toBytes("ccc"))
141        .setEndKey(Bytes.toBytes("ddd"))
142        .build();
143    RegionInfo.add(hri3);
144    regionSizes.put(hri3.getRegionName(), 8);
145
146    RegionInfo hri4 = RegionInfoBuilder.newBuilder(tableName)
147        .setStartKey(Bytes.toBytes("ddd"))
148        .setEndKey(Bytes.toBytes("eee"))
149        .build();
150    regionSizes.put(hri4.getRegionName(), 10);
151
152    setupMocksForNormalizer(regionSizes, RegionInfo);
153    List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
154    assertTrue(plans == null);
155  }
156
157  @Test
158  public void testMergeOfSmallRegions() throws HBaseIOException {
159    final TableName tableName = TableName.valueOf(name.getMethodName());
160    List<RegionInfo> RegionInfo = new ArrayList<>();
161    Map<byte[], Integer> regionSizes = new HashMap<>();
162
163    RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
164        .setStartKey(Bytes.toBytes("aaa"))
165        .setEndKey(Bytes.toBytes("bbb"))
166        .build();
167    RegionInfo.add(hri1);
168    regionSizes.put(hri1.getRegionName(), 15);
169
170    RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
171        .setStartKey(Bytes.toBytes("bbb"))
172        .setEndKey(Bytes.toBytes("ccc"))
173        .build();
174    RegionInfo.add(hri2);
175    regionSizes.put(hri2.getRegionName(), 5);
176
177    RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName)
178        .setStartKey(Bytes.toBytes("ccc"))
179        .setEndKey(Bytes.toBytes("ddd"))
180        .build();
181    RegionInfo.add(hri3);
182    regionSizes.put(hri3.getRegionName(), 5);
183
184    RegionInfo hri4 = RegionInfoBuilder.newBuilder(tableName)
185        .setStartKey(Bytes.toBytes("ddd"))
186        .setEndKey(Bytes.toBytes("eee"))
187        .build();
188    RegionInfo.add(hri4);
189    regionSizes.put(hri4.getRegionName(), 15);
190
191    RegionInfo hri5 = RegionInfoBuilder.newBuilder(tableName)
192        .setStartKey(Bytes.toBytes("eee"))
193        .setEndKey(Bytes.toBytes("fff"))
194        .build();
195    RegionInfo.add(hri5);
196    regionSizes.put(hri5.getRegionName(), 16);
197
198    setupMocksForNormalizer(regionSizes, RegionInfo);
199    List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
200
201    NormalizationPlan plan = plans.get(0);
202    assertTrue(plan instanceof MergeNormalizationPlan);
203    assertEquals(hri2, ((MergeNormalizationPlan) plan).getFirstRegion());
204    assertEquals(hri3, ((MergeNormalizationPlan) plan).getSecondRegion());
205  }
206
207  // Test for situation illustrated in HBASE-14867
208  @Test
209  public void testMergeOfSecondSmallestRegions() throws HBaseIOException {
210    final TableName tableName = TableName.valueOf(name.getMethodName());
211    List<RegionInfo> RegionInfo = new ArrayList<>();
212    Map<byte[], Integer> regionSizes = new HashMap<>();
213
214    RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
215        .setStartKey(Bytes.toBytes("aaa"))
216        .setEndKey(Bytes.toBytes("bbb"))
217        .build();
218    RegionInfo.add(hri1);
219    regionSizes.put(hri1.getRegionName(), 1);
220
221    RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
222        .setStartKey(Bytes.toBytes("bbb"))
223        .setEndKey(Bytes.toBytes("ccc"))
224        .build();
225    RegionInfo.add(hri2);
226    regionSizes.put(hri2.getRegionName(), 10000);
227
228    RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName)
229        .setStartKey(Bytes.toBytes("ccc"))
230        .setEndKey(Bytes.toBytes("ddd"))
231        .build();
232    RegionInfo.add(hri3);
233    regionSizes.put(hri3.getRegionName(), 10000);
234
235    RegionInfo hri4 = RegionInfoBuilder.newBuilder(tableName)
236        .setStartKey(Bytes.toBytes("ddd"))
237        .setEndKey(Bytes.toBytes("eee"))
238        .build();
239    RegionInfo.add(hri4);
240    regionSizes.put(hri4.getRegionName(), 10000);
241
242    RegionInfo hri5 = RegionInfoBuilder.newBuilder(tableName)
243        .setStartKey(Bytes.toBytes("eee"))
244        .setEndKey(Bytes.toBytes("fff"))
245        .build();
246    RegionInfo.add(hri5);
247    regionSizes.put(hri5.getRegionName(), 2700);
248
249    RegionInfo hri6 = RegionInfoBuilder.newBuilder(tableName)
250        .setStartKey(Bytes.toBytes("fff"))
251        .setEndKey(Bytes.toBytes("ggg"))
252        .build();
253    RegionInfo.add(hri6);
254    regionSizes.put(hri6.getRegionName(), 2700);
255
256    setupMocksForNormalizer(regionSizes, RegionInfo);
257    List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
258    NormalizationPlan plan = plans.get(0);
259
260    assertTrue(plan instanceof MergeNormalizationPlan);
261    assertEquals(hri5, ((MergeNormalizationPlan) plan).getFirstRegion());
262    assertEquals(hri6, ((MergeNormalizationPlan) plan).getSecondRegion());
263  }
264
265  @Test
266  public void testMergeOfSmallNonAdjacentRegions() throws HBaseIOException {
267    final TableName tableName = TableName.valueOf(name.getMethodName());
268    List<RegionInfo> RegionInfo = new ArrayList<>();
269    Map<byte[], Integer> regionSizes = new HashMap<>();
270
271    RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
272        .setStartKey(Bytes.toBytes("aaa"))
273        .setEndKey(Bytes.toBytes("bbb"))
274        .build();
275    RegionInfo.add(hri1);
276    regionSizes.put(hri1.getRegionName(), 15);
277
278    RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
279        .setStartKey(Bytes.toBytes("bbb"))
280        .setEndKey(Bytes.toBytes("ccc"))
281        .build();
282    RegionInfo.add(hri2);
283    regionSizes.put(hri2.getRegionName(), 5);
284
285    RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName)
286        .setStartKey(Bytes.toBytes("ccc"))
287        .setEndKey(Bytes.toBytes("ddd"))
288        .build();
289    RegionInfo.add(hri3);
290    regionSizes.put(hri3.getRegionName(), 16);
291
292    RegionInfo hri4 = RegionInfoBuilder.newBuilder(tableName)
293        .setStartKey(Bytes.toBytes("ddd"))
294        .setEndKey(Bytes.toBytes("eee"))
295        .build();
296    RegionInfo.add(hri4);
297    regionSizes.put(hri4.getRegionName(), 15);
298
299    RegionInfo hri5 = RegionInfoBuilder.newBuilder(tableName)
300        .setStartKey(Bytes.toBytes("ddd"))
301        .setEndKey(Bytes.toBytes("eee"))
302        .build();
303    RegionInfo.add(hri4);
304    regionSizes.put(hri5.getRegionName(), 5);
305
306    setupMocksForNormalizer(regionSizes, RegionInfo);
307    List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
308
309    assertTrue(plans == null);
310  }
311
312  @Test
313  public void testSplitOfLargeRegion() throws HBaseIOException {
314    final TableName tableName = TableName.valueOf(name.getMethodName());
315    List<RegionInfo> RegionInfo = new ArrayList<>();
316    Map<byte[], Integer> regionSizes = new HashMap<>();
317
318    RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
319        .setStartKey(Bytes.toBytes("aaa"))
320        .setEndKey(Bytes.toBytes("bbb"))
321        .build();
322    RegionInfo.add(hri1);
323    regionSizes.put(hri1.getRegionName(), 8);
324
325    RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
326        .setStartKey(Bytes.toBytes("bbb"))
327        .setEndKey(Bytes.toBytes("ccc"))
328        .build();
329    RegionInfo.add(hri2);
330    regionSizes.put(hri2.getRegionName(), 6);
331
332    RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName)
333        .setStartKey(Bytes.toBytes("ccc"))
334        .setEndKey(Bytes.toBytes("ddd"))
335        .build();
336    RegionInfo.add(hri3);
337    regionSizes.put(hri3.getRegionName(), 10);
338
339    RegionInfo hri4 = RegionInfoBuilder.newBuilder(tableName)
340        .setStartKey(Bytes.toBytes("ddd"))
341        .setEndKey(Bytes.toBytes("eee"))
342        .build();
343    RegionInfo.add(hri4);
344    regionSizes.put(hri4.getRegionName(), 30);
345
346    setupMocksForNormalizer(regionSizes, RegionInfo);
347    List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
348    NormalizationPlan plan = plans.get(0);
349
350    assertTrue(plan instanceof SplitNormalizationPlan);
351    assertEquals(hri4, ((SplitNormalizationPlan) plan).getRegionInfo());
352  }
353
354  @Test
355  public void testSplitWithTargetRegionCount() throws Exception {
356    final TableName tableName = TableName.valueOf(name.getMethodName());
357    List<RegionInfo> RegionInfo = new ArrayList<>();
358    Map<byte[], Integer> regionSizes = new HashMap<>();
359
360    RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("aaa"))
361        .setEndKey(Bytes.toBytes("bbb")).build();
362    RegionInfo.add(hri1);
363    regionSizes.put(hri1.getRegionName(), 20);
364
365    RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("bbb"))
366        .setEndKey(Bytes.toBytes("ccc")).build();
367    RegionInfo.add(hri2);
368    regionSizes.put(hri2.getRegionName(), 40);
369
370    RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("ccc"))
371        .setEndKey(Bytes.toBytes("ddd")).build();
372    RegionInfo.add(hri3);
373    regionSizes.put(hri3.getRegionName(), 60);
374
375    RegionInfo hri4 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("ddd"))
376        .setEndKey(Bytes.toBytes("eee")).build();
377    RegionInfo.add(hri4);
378    regionSizes.put(hri4.getRegionName(), 80);
379
380    RegionInfo hri5 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("eee"))
381        .setEndKey(Bytes.toBytes("fff")).build();
382    RegionInfo.add(hri5);
383    regionSizes.put(hri5.getRegionName(), 100);
384
385    RegionInfo hri6 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("fff"))
386        .setEndKey(Bytes.toBytes("ggg")).build();
387    RegionInfo.add(hri6);
388    regionSizes.put(hri6.getRegionName(), 120);
389
390    setupMocksForNormalizer(regionSizes, RegionInfo);
391
392    // test when target region size is 20
393    when(masterServices.getTableDescriptors().get(any()).getNormalizerTargetRegionSize())
394        .thenReturn(20L);
395    List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
396    Assert.assertEquals(4, plans.size());
397
398    for (NormalizationPlan plan : plans) {
399      assertTrue(plan instanceof SplitNormalizationPlan);
400    }
401
402    // test when target region size is 200
403    when(masterServices.getTableDescriptors().get(any()).getNormalizerTargetRegionSize())
404        .thenReturn(200L);
405    plans = normalizer.computePlanForTable(tableName);
406    Assert.assertEquals(2, plans.size());
407    NormalizationPlan plan = plans.get(0);
408    assertTrue(plan instanceof MergeNormalizationPlan);
409    assertEquals(hri1, ((MergeNormalizationPlan) plan).getFirstRegion());
410    assertEquals(hri2, ((MergeNormalizationPlan) plan).getSecondRegion());
411  }
412
413  @Test
414  public void testSplitWithTargetRegionSize() throws Exception {
415    final TableName tableName = TableName.valueOf(name.getMethodName());
416    List<RegionInfo> RegionInfo = new ArrayList<>();
417    Map<byte[], Integer> regionSizes = new HashMap<>();
418
419    RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("aaa"))
420        .setEndKey(Bytes.toBytes("bbb")).build();
421    RegionInfo.add(hri1);
422    regionSizes.put(hri1.getRegionName(), 20);
423
424    RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("bbb"))
425        .setEndKey(Bytes.toBytes("ccc")).build();
426    RegionInfo.add(hri2);
427    regionSizes.put(hri2.getRegionName(), 40);
428
429    RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("ccc"))
430        .setEndKey(Bytes.toBytes("ddd")).build();
431    RegionInfo.add(hri3);
432    regionSizes.put(hri3.getRegionName(), 60);
433
434    RegionInfo hri4 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("ddd"))
435        .setEndKey(Bytes.toBytes("eee")).build();
436    RegionInfo.add(hri4);
437    regionSizes.put(hri4.getRegionName(), 80);
438
439    setupMocksForNormalizer(regionSizes, RegionInfo);
440
441    // test when target region count is 8
442    when(masterServices.getTableDescriptors().get(any()).getNormalizerTargetRegionCount())
443        .thenReturn(8);
444    List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
445    Assert.assertEquals(2, plans.size());
446
447    for (NormalizationPlan plan : plans) {
448      assertTrue(plan instanceof SplitNormalizationPlan);
449    }
450
451    // test when target region count is 3
452    when(masterServices.getTableDescriptors().get(any()).getNormalizerTargetRegionCount())
453        .thenReturn(3);
454    plans = normalizer.computePlanForTable(tableName);
455    Assert.assertEquals(1, plans.size());
456    NormalizationPlan plan = plans.get(0);
457    assertTrue(plan instanceof MergeNormalizationPlan);
458    assertEquals(hri1, ((MergeNormalizationPlan) plan).getFirstRegion());
459    assertEquals(hri2, ((MergeNormalizationPlan) plan).getSecondRegion());
460  }
461
462  @SuppressWarnings("MockitoCast")
463  protected void setupMocksForNormalizer(Map<byte[], Integer> regionSizes,
464                                         List<RegionInfo> RegionInfo) {
465    masterServices = Mockito.mock(MasterServices.class, RETURNS_DEEP_STUBS);
466    masterRpcServices = Mockito.mock(MasterRpcServices.class, RETURNS_DEEP_STUBS);
467
468    // for simplicity all regions are assumed to be on one server; doesn't matter to us
469    ServerName sn = ServerName.valueOf("localhost", 0, 1L);
470    when(masterServices.getAssignmentManager().getRegionStates().
471      getRegionsOfTable(any())).thenReturn(RegionInfo);
472    when(masterServices.getAssignmentManager().getRegionStates().
473      getRegionServerOfRegion(any())).thenReturn(sn);
474
475    for (Map.Entry<byte[], Integer> region : regionSizes.entrySet()) {
476      RegionMetrics regionLoad = Mockito.mock(RegionMetrics.class);
477      when(regionLoad.getRegionName()).thenReturn(region.getKey());
478      when(regionLoad.getStoreFileSize())
479        .thenReturn(new Size(region.getValue(), Size.Unit.MEGABYTE));
480
481      // this is possibly broken with jdk9, unclear if false positive or not
482      // suppress it for now, fix it when we get to running tests on 9
483      // see: http://errorprone.info/bugpattern/MockitoCast
484      when((Object) masterServices.getServerManager().getLoad(sn).
485        getRegionMetrics().get(region.getKey())).thenReturn(regionLoad);
486    }
487    try {
488      when(masterRpcServices.isSplitOrMergeEnabled(any(),
489        any())).thenReturn(
490          IsSplitOrMergeEnabledResponse.newBuilder().setEnabled(true).build());
491    } catch (ServiceException se) {
492      LOG.debug("error setting isSplitOrMergeEnabled switch", se);
493    }
494
495    normalizer.setMasterServices(masterServices);
496    normalizer.setMasterRpcServices(masterRpcServices);
497  }
498}