001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024import static org.mockito.Mockito.doReturn;
025import static org.mockito.Mockito.mock;
026import static org.mockito.Mockito.when;
027
028import java.io.IOException;
029import java.util.ArrayList;
030import java.util.List;
031import java.util.Optional;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseConfiguration;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.client.RegionInfoBuilder;
039import org.apache.hadoop.hbase.client.TableDescriptor;
040import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
041import org.apache.hadoop.hbase.testclassification.RegionServerTests;
042import org.apache.hadoop.hbase.testclassification.SmallTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
045import org.junit.Before;
046import org.junit.ClassRule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049
050@Category({ RegionServerTests.class, SmallTests.class })
051public class TestRegionSplitPolicy {
052
053  @ClassRule
054  public static final HBaseClassTestRule CLASS_RULE =
055    HBaseClassTestRule.forClass(TestRegionSplitPolicy.class);
056
057  private Configuration conf;
058  private HRegion mockRegion;
059  private List<HStore> stores;
060  private static final TableName TABLENAME = TableName.valueOf("t");
061
062  @Before
063  public void setupMocks() {
064    conf = HBaseConfiguration.create();
065    RegionInfo hri = RegionInfoBuilder.newBuilder(TABLENAME).build();
066    mockRegion = mock(HRegion.class);
067    doReturn(hri).when(mockRegion).getRegionInfo();
068    doReturn(true).when(mockRegion).isAvailable();
069    stores = new ArrayList<>();
070    doReturn(stores).when(mockRegion).getStores();
071  }
072
073  @Test
074  public void testForceSplitRegionWithReference() throws IOException {
075    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(1024L).build();
076    doReturn(td).when(mockRegion).getTableDescriptor();
077    // Add a store above the requisite size. Should split.
078    HStore mockStore = mock(HStore.class);
079    doReturn(2000L).when(mockStore).getSize();
080    // Act as if there's a reference file or some other reason it can't split.
081    // This should prevent splitting even though it's big enough.
082    doReturn(false).when(mockStore).canSplit();
083    stores.add(mockStore);
084
085    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
086      ConstantSizeRegionSplitPolicy.class.getName());
087    ConstantSizeRegionSplitPolicy policy =
088      (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
089    assertFalse(policy.shouldSplit());
090
091    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
092      IncreasingToUpperBoundRegionSplitPolicy.class.getName());
093    policy = (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
094    assertFalse(policy.shouldSplit());
095  }
096
097  @Test
098  public void testIncreasingToUpperBoundRegionSplitPolicy() throws IOException {
099    // Configure IncreasingToUpperBoundRegionSplitPolicy as our split policy
100    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
101      IncreasingToUpperBoundRegionSplitPolicy.class.getName());
102    // Now make it so the mock region has a RegionServerService that will
103    // return 'online regions'.
104    RegionServerServices rss = mock(RegionServerServices.class);
105    final List<HRegion> regions = new ArrayList<>();
106    doReturn(regions).when(rss).getRegions(TABLENAME);
107    when(mockRegion.getRegionServerServices()).thenReturn(rss);
108    // Set max size for this 'table'.
109    long maxSplitSize = 1024L;
110    // Set flush size to 1/8. IncreasingToUpperBoundRegionSplitPolicy
111    // grows by the cube of the number of regions times flushsize each time.
112    long flushSize = maxSplitSize / 8;
113    conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSize);
114    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(maxSplitSize)
115      .setMemStoreFlushSize(flushSize).build();
116    doReturn(td).when(mockRegion).getTableDescriptor();
117    // If RegionServerService with no regions in it -- 'online regions' == 0 --
118    // then IncreasingToUpperBoundRegionSplitPolicy should act like a
119    // ConstantSizePolicy
120    IncreasingToUpperBoundRegionSplitPolicy policy =
121      (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
122    doConstantSizePolicyTests(policy);
123
124    // Add a store in excess of split size. Because there are "no regions"
125    // on this server -- rss.getOnlineRegions is 0 -- then we should split
126    // like a constantsizeregionsplitpolicy would
127    HStore mockStore = mock(HStore.class);
128    doReturn(2000L).when(mockStore).getSize();
129    doReturn(true).when(mockStore).canSplit();
130    stores.add(mockStore);
131    // It should split
132    assertTrue(policy.shouldSplit());
133
134    // Now test that we increase our split size as online regions for a table
135    // grows. With one region, split size should be flushsize.
136    regions.add(mockRegion);
137    doReturn(flushSize).when(mockStore).getSize();
138    // Should not split since store is flush size.
139    assertFalse(policy.shouldSplit());
140    // Set size of store to be > 2*flush size and we should split
141    doReturn(flushSize * 2 + 1).when(mockStore).getSize();
142    assertTrue(policy.shouldSplit());
143    // Add another region to the 'online regions' on this server and we should
144    // now be no longer be splittable since split size has gone up.
145    regions.add(mockRegion);
146    assertFalse(policy.shouldSplit());
147    // make sure its just over; verify it'll split
148    doReturn((long) (maxSplitSize * 1.25 + 1)).when(mockStore).getSize();
149    assertTrue(policy.shouldSplit());
150
151    // Finally assert that even if loads of regions, we'll split at max size
152    assertWithinJitter(maxSplitSize, policy.getSizeToCheck(1000));
153    // Assert same is true if count of regions is zero.
154    assertWithinJitter(maxSplitSize, policy.getSizeToCheck(0));
155  }
156
157  @Test
158  public void testIsExceedSize() throws IOException {
159    // Configure SteppingAllStoresSizeSplitPolicy as our split policy
160    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
161      ConstantSizeRegionSplitPolicy.class.getName());
162    // Now make it so the mock region has a RegionServerService that will
163    // return 'online regions'.
164    RegionServerServices rss = mock(RegionServerServices.class);
165    final List<HRegion> regions = new ArrayList<>();
166    doReturn(regions).when(rss).getRegions(TABLENAME);
167    when(mockRegion.getRegionServerServices()).thenReturn(rss);
168
169    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).build();
170    doReturn(td).when(mockRegion).getTableDescriptor();
171    ConstantSizeRegionSplitPolicy policy =
172      (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
173    regions.add(mockRegion);
174
175    HStore mockStore1 = mock(HStore.class);
176    doReturn(100L).when(mockStore1).getSize();
177    HStore mockStore2 = mock(HStore.class);
178    doReturn(924L).when(mockStore2).getSize();
179    HStore mockStore3 = mock(HStore.class);
180    doReturn(925L).when(mockStore3).getSize();
181
182    // test sum of store's size not greater than sizeToCheck
183    stores.add(mockStore1);
184    stores.add(mockStore2);
185    assertFalse(policy.isExceedSize(1024));
186    stores.clear();
187
188    // test sum of store's size greater than sizeToCheck
189    stores.add(mockStore1);
190    stores.add(mockStore3);
191    assertTrue(policy.isExceedSize(1024));
192  }
193
194  @Test
195  public void testBusyRegionSplitPolicy() throws Exception {
196    doReturn(TableDescriptorBuilder.newBuilder(TABLENAME).build()).when(mockRegion)
197      .getTableDescriptor();
198    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, BusyRegionSplitPolicy.class.getName());
199    conf.setLong("hbase.busy.policy.minAge", 1000000L);
200    conf.setFloat("hbase.busy.policy.blockedRequests", 0.1f);
201
202    RegionServerServices rss = mock(RegionServerServices.class);
203    final List<HRegion> regions = new ArrayList<>();
204    doReturn(regions).when(rss).getRegions(TABLENAME);
205    when(mockRegion.getRegionServerServices()).thenReturn(rss);
206    when(mockRegion.getBlockedRequestsCount()).thenReturn(0L);
207    when(mockRegion.getWriteRequestsCount()).thenReturn(0L);
208
209    BusyRegionSplitPolicy policy =
210      (BusyRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
211
212    when(mockRegion.getBlockedRequestsCount()).thenReturn(10L);
213    when(mockRegion.getWriteRequestsCount()).thenReturn(10L);
214    // Not enough time since region came online
215    assertFalse(policy.shouldSplit());
216
217    // Reset min age for split to zero
218    conf.setLong("hbase.busy.policy.minAge", 0L);
219    // Aggregate over 500 ms periods
220    conf.setLong("hbase.busy.policy.aggWindow", 500L);
221    policy = (BusyRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
222    long start = EnvironmentEdgeManager.currentTime();
223    when(mockRegion.getBlockedRequestsCount()).thenReturn(10L);
224    when(mockRegion.getWriteRequestsCount()).thenReturn(20L);
225    Thread.sleep(300);
226    assertFalse(policy.shouldSplit());
227    when(mockRegion.getBlockedRequestsCount()).thenReturn(12L);
228    when(mockRegion.getWriteRequestsCount()).thenReturn(30L);
229    Thread.sleep(2);
230    // Enough blocked requests since last time, but aggregate blocked request
231    // rate over last 500 ms is still low, because major portion of the window is constituted
232    // by the previous zero blocked request period which lasted at least 300 ms off last 500 ms.
233    if (EnvironmentEdgeManager.currentTime() - start < 500) {
234      assertFalse(policy.shouldSplit());
235    }
236    when(mockRegion.getBlockedRequestsCount()).thenReturn(14L);
237    when(mockRegion.getWriteRequestsCount()).thenReturn(40L);
238    Thread.sleep(200);
239    assertTrue(policy.shouldSplit());
240  }
241
242  private void assertWithinJitter(long maxSplitSize, long sizeToCheck) {
243    assertTrue("Size greater than lower bound of jitter",
244      (long) (maxSplitSize * 0.75) <= sizeToCheck);
245    assertTrue("Size less than upper bound of jitter", (long) (maxSplitSize * 1.25) >= sizeToCheck);
246  }
247
248  @Test
249  public void testCreateDefault() throws IOException {
250    conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L);
251    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).build();
252    doReturn(td).when(mockRegion).getTableDescriptor();
253    // Using a default HTD, should pick up the file size from
254    // configuration.
255    ConstantSizeRegionSplitPolicy policy =
256      (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
257    assertWithinJitter(1234L, policy.getDesiredMaxFileSize());
258
259    // If specified in HTD, should use that
260    td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(9999L).build();
261    doReturn(td).when(mockRegion).getTableDescriptor();
262    policy = (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
263    assertWithinJitter(9999L, policy.getDesiredMaxFileSize());
264  }
265
266  /**
267   * Test setting up a customized split policy
268   */
269  @Test
270  public void testCustomPolicy() throws IOException {
271    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME)
272      .setRegionSplitPolicyClassName(KeyPrefixRegionSplitPolicy.class.getName())
273      .setValue(KeyPrefixRegionSplitPolicy.PREFIX_LENGTH_KEY, "2").build();
274
275    doReturn(td).when(mockRegion).getTableDescriptor();
276
277    HStore mockStore = mock(HStore.class);
278    doReturn(2000L).when(mockStore).getSize();
279    doReturn(true).when(mockStore).canSplit();
280    doReturn(Optional.of(Bytes.toBytes("abcd"))).when(mockStore).getSplitPoint();
281    stores.add(mockStore);
282
283    KeyPrefixRegionSplitPolicy policy =
284      (KeyPrefixRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
285
286    assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
287  }
288
289  @Test
290  public void testConstantSizePolicy() throws IOException {
291    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(1024L).build();
292    doReturn(td).when(mockRegion).getTableDescriptor();
293    ConstantSizeRegionSplitPolicy policy =
294      (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
295    doConstantSizePolicyTests(policy);
296  }
297
298  /**
299   * Run through tests for a ConstantSizeRegionSplitPolicy
300   */
301  private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) {
302    // For no stores, should not split
303    assertFalse(policy.shouldSplit());
304
305    // Add a store above the requisite size. Should split.
306    HStore mockStore = mock(HStore.class);
307    doReturn(2000L).when(mockStore).getSize();
308    doReturn(true).when(mockStore).canSplit();
309    stores.add(mockStore);
310
311    assertTrue(policy.shouldSplit());
312
313    // Act as if there's a reference file or some other reason it can't split.
314    // This should prevent splitting even though it's big enough.
315    doReturn(false).when(mockStore).canSplit();
316    assertFalse(policy.shouldSplit());
317
318    // Reset splittability after above
319    doReturn(true).when(mockStore).canSplit();
320
321    // Set to a small size, should not split
322    doReturn(100L).when(mockStore).getSize();
323    assertFalse(policy.shouldSplit());
324
325    // Clear families we added above
326    stores.clear();
327  }
328
329  @Test
330  public void testGetSplitPoint() throws IOException {
331    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).build();
332    doReturn(td).when(mockRegion).getTableDescriptor();
333
334    ConstantSizeRegionSplitPolicy policy =
335      (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
336
337    // For no stores, should not split
338    assertFalse(policy.shouldSplit());
339    assertNull(policy.getSplitPoint());
340
341    // Add a store above the requisite size. Should split.
342    HStore mockStore = mock(HStore.class);
343    doReturn(2000L).when(mockStore).getSize();
344    doReturn(true).when(mockStore).canSplit();
345    doReturn(Optional.of(Bytes.toBytes("store 1 split"))).when(mockStore).getSplitPoint();
346    stores.add(mockStore);
347
348    assertEquals("store 1 split", Bytes.toString(policy.getSplitPoint()));
349
350    // Add a bigger store. The split point should come from that one
351    HStore mockStore2 = mock(HStore.class);
352    doReturn(4000L).when(mockStore2).getSize();
353    doReturn(true).when(mockStore2).canSplit();
354    doReturn(Optional.of(Bytes.toBytes("store 2 split"))).when(mockStore2).getSplitPoint();
355    stores.add(mockStore2);
356
357    assertEquals("store 2 split", Bytes.toString(policy.getSplitPoint()));
358  }
359
360  @Test
361  public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException {
362    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME)
363      .setRegionSplitPolicyClassName(DelimitedKeyPrefixRegionSplitPolicy.class.getName())
364      .setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ",").build();
365
366    doReturn(td).when(mockRegion).getTableDescriptor();
367    doReturn(stores).when(mockRegion).getStores();
368
369    HStore mockStore = mock(HStore.class);
370    doReturn(2000L).when(mockStore).getSize();
371    doReturn(true).when(mockStore).canSplit();
372    doReturn(Optional.of(Bytes.toBytes("ab,cd"))).when(mockStore).getSplitPoint();
373    stores.add(mockStore);
374
375    DelimitedKeyPrefixRegionSplitPolicy policy =
376      (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
377
378    assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
379
380    doReturn(Optional.of(Bytes.toBytes("ijk"))).when(mockStore).getSplitPoint();
381    assertEquals("ijk", Bytes.toString(policy.getSplitPoint()));
382  }
383
384  @Test
385  public void testConstantSizePolicyWithJitter() throws IOException {
386    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
387      ConstantSizeRegionSplitPolicy.class.getName());
388    TableDescriptor td =
389      TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(Long.MAX_VALUE).build();
390    doReturn(td).when(mockRegion).getTableDescriptor();
391    boolean positiveJitter = false;
392    ConstantSizeRegionSplitPolicy policy = null;
393    while (!positiveJitter) {
394      policy = (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
395      positiveJitter = policy.positiveJitterRate();
396    }
397    // add a store
398    HStore mockStore = mock(HStore.class);
399    doReturn(2000L).when(mockStore).getSize();
400    doReturn(true).when(mockStore).canSplit();
401    stores.add(mockStore);
402    // Jitter shouldn't cause overflow when HTableDescriptor.MAX_FILESIZE set to Long.MAX_VALUE
403    assertFalse(policy.shouldSplit());
404  }
405}