Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.normalizer;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.mockito.Matchers.any;
023import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
024import static org.mockito.Mockito.when;
025
026import java.util.ArrayList;
027import java.util.HashMap;
028import java.util.List;
029import java.util.Map;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseIOException;
032import org.apache.hadoop.hbase.RegionMetrics;
033import org.apache.hadoop.hbase.ServerName;
034import org.apache.hadoop.hbase.Size;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.client.RegionInfoBuilder;
038import org.apache.hadoop.hbase.master.MasterRpcServices;
039import org.apache.hadoop.hbase.master.MasterServices;
040import org.apache.hadoop.hbase.testclassification.MasterTests;
041import org.apache.hadoop.hbase.testclassification.SmallTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.junit.BeforeClass;
044import org.junit.ClassRule;
045import org.junit.Rule;
046import org.junit.Test;
047import org.junit.experimental.categories.Category;
048import org.junit.rules.TestName;
049import org.mockito.Mockito;
050import org.slf4j.Logger;
051import org.slf4j.LoggerFactory;
052
053import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
054
055import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledResponse;
056
057/**
058 * Tests logic of {@link SimpleRegionNormalizer}.
059 */
060@Category({MasterTests.class, SmallTests.class})
061public class TestSimpleRegionNormalizer {
062
063  @ClassRule
064  public static final HBaseClassTestRule CLASS_RULE =
065      HBaseClassTestRule.forClass(TestSimpleRegionNormalizer.class);
066
067  private static final Logger LOG = LoggerFactory.getLogger(TestSimpleRegionNormalizer.class);
068
069  private static RegionNormalizer normalizer;
070
071  // mocks
072  private static MasterServices masterServices;
073  private static MasterRpcServices masterRpcServices;
074
075  @Rule
076  public TestName name = new TestName();
077
078  @BeforeClass
079  public static void beforeAllTests() throws Exception {
080    normalizer = new SimpleRegionNormalizer();
081  }
082
083  @Test
084  public void testNoNormalizationForMetaTable() throws HBaseIOException {
085    TableName testTable = TableName.META_TABLE_NAME;
086    List<RegionInfo> RegionInfo = new ArrayList<>();
087    Map<byte[], Integer> regionSizes = new HashMap<>();
088
089    setupMocksForNormalizer(regionSizes, RegionInfo);
090    List<NormalizationPlan> plans = normalizer.computePlanForTable(testTable);
091    assertTrue(plans == null);
092  }
093
094  @Test
095  public void testNoNormalizationIfTooFewRegions() throws HBaseIOException {
096    final TableName tableName = TableName.valueOf(name.getMethodName());
097    List<RegionInfo> RegionInfo = new ArrayList<>();
098    Map<byte[], Integer> regionSizes = new HashMap<>();
099    RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
100        .setStartKey(Bytes.toBytes("aaa"))
101        .setEndKey(Bytes.toBytes("bbb"))
102        .build();
103    RegionInfo.add(hri1);
104    regionSizes.put(hri1.getRegionName(), 10);
105
106    RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
107        .setStartKey(Bytes.toBytes("bbb"))
108        .setEndKey(Bytes.toBytes("ccc"))
109        .build();
110    RegionInfo.add(hri2);
111    regionSizes.put(hri2.getRegionName(), 15);
112
113    setupMocksForNormalizer(regionSizes, RegionInfo);
114    List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
115    assertTrue(plans == null);
116  }
117
118  @Test
119  public void testNoNormalizationOnNormalizedCluster() throws HBaseIOException {
120    final TableName tableName = TableName.valueOf(name.getMethodName());
121    List<RegionInfo> RegionInfo = new ArrayList<>();
122    Map<byte[], Integer> regionSizes = new HashMap<>();
123
124    RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
125        .setStartKey(Bytes.toBytes("aaa"))
126        .setEndKey(Bytes.toBytes("bbb"))
127        .build();
128    RegionInfo.add(hri1);
129    regionSizes.put(hri1.getRegionName(), 10);
130
131    RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
132        .setStartKey(Bytes.toBytes("bbb"))
133        .setEndKey(Bytes.toBytes("ccc"))
134        .build();
135    RegionInfo.add(hri2);
136    regionSizes.put(hri2.getRegionName(), 15);
137
138    RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName)
139        .setStartKey(Bytes.toBytes("ccc"))
140        .setEndKey(Bytes.toBytes("ddd"))
141        .build();
142    RegionInfo.add(hri3);
143    regionSizes.put(hri3.getRegionName(), 8);
144
145    RegionInfo hri4 = RegionInfoBuilder.newBuilder(tableName)
146        .setStartKey(Bytes.toBytes("ddd"))
147        .setEndKey(Bytes.toBytes("eee"))
148        .build();
149    regionSizes.put(hri4.getRegionName(), 10);
150
151    setupMocksForNormalizer(regionSizes, RegionInfo);
152    List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
153    assertTrue(plans == null);
154  }
155
156  @Test
157  public void testMergeOfSmallRegions() throws HBaseIOException {
158    final TableName tableName = TableName.valueOf(name.getMethodName());
159    List<RegionInfo> RegionInfo = new ArrayList<>();
160    Map<byte[], Integer> regionSizes = new HashMap<>();
161
162    RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
163        .setStartKey(Bytes.toBytes("aaa"))
164        .setEndKey(Bytes.toBytes("bbb"))
165        .build();
166    RegionInfo.add(hri1);
167    regionSizes.put(hri1.getRegionName(), 15);
168
169    RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
170        .setStartKey(Bytes.toBytes("bbb"))
171        .setEndKey(Bytes.toBytes("ccc"))
172        .build();
173    RegionInfo.add(hri2);
174    regionSizes.put(hri2.getRegionName(), 5);
175
176    RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName)
177        .setStartKey(Bytes.toBytes("ccc"))
178        .setEndKey(Bytes.toBytes("ddd"))
179        .build();
180    RegionInfo.add(hri3);
181    regionSizes.put(hri3.getRegionName(), 5);
182
183    RegionInfo hri4 = RegionInfoBuilder.newBuilder(tableName)
184        .setStartKey(Bytes.toBytes("ddd"))
185        .setEndKey(Bytes.toBytes("eee"))
186        .build();
187    RegionInfo.add(hri4);
188    regionSizes.put(hri4.getRegionName(), 15);
189
190    RegionInfo hri5 = RegionInfoBuilder.newBuilder(tableName)
191        .setStartKey(Bytes.toBytes("eee"))
192        .setEndKey(Bytes.toBytes("fff"))
193        .build();
194    RegionInfo.add(hri5);
195    regionSizes.put(hri5.getRegionName(), 16);
196
197    setupMocksForNormalizer(regionSizes, RegionInfo);
198    List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
199
200    NormalizationPlan plan = plans.get(0);
201    assertTrue(plan instanceof MergeNormalizationPlan);
202    assertEquals(hri2, ((MergeNormalizationPlan) plan).getFirstRegion());
203    assertEquals(hri3, ((MergeNormalizationPlan) plan).getSecondRegion());
204  }
205
206  // Test for situation illustrated in HBASE-14867
207  @Test
208  public void testMergeOfSecondSmallestRegions() throws HBaseIOException {
209    final TableName tableName = TableName.valueOf(name.getMethodName());
210    List<RegionInfo> RegionInfo = new ArrayList<>();
211    Map<byte[], Integer> regionSizes = new HashMap<>();
212
213    RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
214        .setStartKey(Bytes.toBytes("aaa"))
215        .setEndKey(Bytes.toBytes("bbb"))
216        .build();
217    RegionInfo.add(hri1);
218    regionSizes.put(hri1.getRegionName(), 1);
219
220    RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
221        .setStartKey(Bytes.toBytes("bbb"))
222        .setEndKey(Bytes.toBytes("ccc"))
223        .build();
224    RegionInfo.add(hri2);
225    regionSizes.put(hri2.getRegionName(), 10000);
226
227    RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName)
228        .setStartKey(Bytes.toBytes("ccc"))
229        .setEndKey(Bytes.toBytes("ddd"))
230        .build();
231    RegionInfo.add(hri3);
232    regionSizes.put(hri3.getRegionName(), 10000);
233
234    RegionInfo hri4 = RegionInfoBuilder.newBuilder(tableName)
235        .setStartKey(Bytes.toBytes("ddd"))
236        .setEndKey(Bytes.toBytes("eee"))
237        .build();
238    RegionInfo.add(hri4);
239    regionSizes.put(hri4.getRegionName(), 10000);
240
241    RegionInfo hri5 = RegionInfoBuilder.newBuilder(tableName)
242        .setStartKey(Bytes.toBytes("eee"))
243        .setEndKey(Bytes.toBytes("fff"))
244        .build();
245    RegionInfo.add(hri5);
246    regionSizes.put(hri5.getRegionName(), 2700);
247
248    RegionInfo hri6 = RegionInfoBuilder.newBuilder(tableName)
249        .setStartKey(Bytes.toBytes("fff"))
250        .setEndKey(Bytes.toBytes("ggg"))
251        .build();
252    RegionInfo.add(hri6);
253    regionSizes.put(hri6.getRegionName(), 2700);
254
255    setupMocksForNormalizer(regionSizes, RegionInfo);
256    List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
257    NormalizationPlan plan = plans.get(0);
258
259    assertTrue(plan instanceof MergeNormalizationPlan);
260    assertEquals(hri5, ((MergeNormalizationPlan) plan).getFirstRegion());
261    assertEquals(hri6, ((MergeNormalizationPlan) plan).getSecondRegion());
262  }
263
264  @Test
265  public void testMergeOfSmallNonAdjacentRegions() throws HBaseIOException {
266    final TableName tableName = TableName.valueOf(name.getMethodName());
267    List<RegionInfo> RegionInfo = new ArrayList<>();
268    Map<byte[], Integer> regionSizes = new HashMap<>();
269
270    RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
271        .setStartKey(Bytes.toBytes("aaa"))
272        .setEndKey(Bytes.toBytes("bbb"))
273        .build();
274    RegionInfo.add(hri1);
275    regionSizes.put(hri1.getRegionName(), 15);
276
277    RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
278        .setStartKey(Bytes.toBytes("bbb"))
279        .setEndKey(Bytes.toBytes("ccc"))
280        .build();
281    RegionInfo.add(hri2);
282    regionSizes.put(hri2.getRegionName(), 5);
283
284    RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName)
285        .setStartKey(Bytes.toBytes("ccc"))
286        .setEndKey(Bytes.toBytes("ddd"))
287        .build();
288    RegionInfo.add(hri3);
289    regionSizes.put(hri3.getRegionName(), 16);
290
291    RegionInfo hri4 = RegionInfoBuilder.newBuilder(tableName)
292        .setStartKey(Bytes.toBytes("ddd"))
293        .setEndKey(Bytes.toBytes("eee"))
294        .build();
295    RegionInfo.add(hri4);
296    regionSizes.put(hri4.getRegionName(), 15);
297
298    RegionInfo hri5 = RegionInfoBuilder.newBuilder(tableName)
299        .setStartKey(Bytes.toBytes("ddd"))
300        .setEndKey(Bytes.toBytes("eee"))
301        .build();
302    RegionInfo.add(hri4);
303    regionSizes.put(hri5.getRegionName(), 5);
304
305    setupMocksForNormalizer(regionSizes, RegionInfo);
306    List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
307
308    assertTrue(plans == null);
309  }
310
311  @Test
312  public void testSplitOfLargeRegion() throws HBaseIOException {
313    final TableName tableName = TableName.valueOf(name.getMethodName());
314    List<RegionInfo> RegionInfo = new ArrayList<>();
315    Map<byte[], Integer> regionSizes = new HashMap<>();
316
317    RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
318        .setStartKey(Bytes.toBytes("aaa"))
319        .setEndKey(Bytes.toBytes("bbb"))
320        .build();
321    RegionInfo.add(hri1);
322    regionSizes.put(hri1.getRegionName(), 8);
323
324    RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
325        .setStartKey(Bytes.toBytes("bbb"))
326        .setEndKey(Bytes.toBytes("ccc"))
327        .build();
328    RegionInfo.add(hri2);
329    regionSizes.put(hri2.getRegionName(), 6);
330
331    RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName)
332        .setStartKey(Bytes.toBytes("ccc"))
333        .setEndKey(Bytes.toBytes("ddd"))
334        .build();
335    RegionInfo.add(hri3);
336    regionSizes.put(hri3.getRegionName(), 10);
337
338    RegionInfo hri4 = RegionInfoBuilder.newBuilder(tableName)
339        .setStartKey(Bytes.toBytes("ddd"))
340        .setEndKey(Bytes.toBytes("eee"))
341        .build();
342    RegionInfo.add(hri4);
343    regionSizes.put(hri4.getRegionName(), 30);
344
345    setupMocksForNormalizer(regionSizes, RegionInfo);
346    List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
347    NormalizationPlan plan = plans.get(0);
348
349    assertTrue(plan instanceof SplitNormalizationPlan);
350    assertEquals(hri4, ((SplitNormalizationPlan) plan).getRegionInfo());
351  }
352
353  @SuppressWarnings("MockitoCast")
354  protected void setupMocksForNormalizer(Map<byte[], Integer> regionSizes,
355                                         List<RegionInfo> RegionInfo) {
356    masterServices = Mockito.mock(MasterServices.class, RETURNS_DEEP_STUBS);
357    masterRpcServices = Mockito.mock(MasterRpcServices.class, RETURNS_DEEP_STUBS);
358
359    // for simplicity all regions are assumed to be on one server; doesn't matter to us
360    ServerName sn = ServerName.valueOf("localhost", 0, 1L);
361    when(masterServices.getAssignmentManager().getRegionStates().
362      getRegionsOfTable(any())).thenReturn(RegionInfo);
363    when(masterServices.getAssignmentManager().getRegionStates().
364      getRegionServerOfRegion(any())).thenReturn(sn);
365
366    for (Map.Entry<byte[], Integer> region : regionSizes.entrySet()) {
367      RegionMetrics regionLoad = Mockito.mock(RegionMetrics.class);
368      when(regionLoad.getRegionName()).thenReturn(region.getKey());
369      when(regionLoad.getStoreFileSize())
370        .thenReturn(new Size(region.getValue(), Size.Unit.MEGABYTE));
371
372      // this is possibly broken with jdk9, unclear if false positive or not
373      // suppress it for now, fix it when we get to running tests on 9
374      // see: http://errorprone.info/bugpattern/MockitoCast
375      when((Object) masterServices.getServerManager().getLoad(sn).
376        getRegionMetrics().get(region.getKey())).thenReturn(regionLoad);
377    }
378    try {
379      when(masterRpcServices.isSplitOrMergeEnabled(any(),
380        any())).thenReturn(
381          IsSplitOrMergeEnabledResponse.newBuilder().setEnabled(true).build());
382    } catch (ServiceException se) {
383      LOG.debug("error setting isSplitOrMergeEnabled switch", se);
384    }
385
386    normalizer.setMasterServices(masterServices);
387    normalizer.setMasterRpcServices(masterRpcServices);
388  }
389}