001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.List;
028import java.util.Optional;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseConfiguration;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.HRegionInfo;
034import org.apache.hadoop.hbase.HTableDescriptor;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.testclassification.RegionServerTests;
037import org.apache.hadoop.hbase.testclassification.SmallTests;
038import org.apache.hadoop.hbase.util.Bytes;
039import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
040import org.junit.Before;
041import org.junit.ClassRule;
042import org.junit.Rule;
043import org.junit.Test;
044import org.junit.experimental.categories.Category;
045import org.junit.rules.TestName;
046import org.mockito.Mockito;
047
048@Category({RegionServerTests.class, SmallTests.class})
049public class TestRegionSplitPolicy {
050
051  @ClassRule
052  public static final HBaseClassTestRule CLASS_RULE =
053      HBaseClassTestRule.forClass(TestRegionSplitPolicy.class);
054
055  private Configuration conf;
056  private HTableDescriptor htd;
057  private HRegion mockRegion;
058  private List<HStore> stores;
059  private static final TableName TABLENAME = TableName.valueOf("t");
060
061  @Rule
062  public TestName name = new TestName();
063
064  @Before
065  public void setupMocks() {
066    conf = HBaseConfiguration.create();
067    HRegionInfo hri = new HRegionInfo(TABLENAME);
068    htd = new HTableDescriptor(TABLENAME);
069    mockRegion = Mockito.mock(HRegion.class);
070    Mockito.doReturn(htd).when(mockRegion).getTableDescriptor();
071    Mockito.doReturn(hri).when(mockRegion).getRegionInfo();
072    stores = new ArrayList<>();
073    Mockito.doReturn(stores).when(mockRegion).getStores();
074  }
075
076  @Test
077  public void testForceSplitRegionWithReference() throws IOException {
078    htd.setMaxFileSize(1024L);
079    // Add a store above the requisite size. Should split.
080    HStore mockStore = Mockito.mock(HStore.class);
081    Mockito.doReturn(2000L).when(mockStore).getSize();
082    // Act as if there's a reference file or some other reason it can't split.
083    // This should prevent splitting even though it's big enough.
084    Mockito.doReturn(false).when(mockStore).canSplit();
085    stores.add(mockStore);
086
087    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
088      ConstantSizeRegionSplitPolicy.class.getName());
089    ConstantSizeRegionSplitPolicy policy =
090        (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
091    assertFalse(policy.shouldSplit());
092    Mockito.doReturn(true).when(mockRegion).shouldForceSplit();
093    assertFalse(policy.shouldSplit());
094
095    Mockito.doReturn(false).when(mockRegion).shouldForceSplit();
096    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
097      IncreasingToUpperBoundRegionSplitPolicy.class.getName());
098    policy = (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
099    assertFalse(policy.shouldSplit());
100    Mockito.doReturn(true).when(mockRegion).shouldForceSplit();
101    assertFalse(policy.shouldSplit());
102  }
103
104  @Test
105  public void testIncreasingToUpperBoundRegionSplitPolicy() throws IOException {
106    // Configure IncreasingToUpperBoundRegionSplitPolicy as our split policy
107    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
108      IncreasingToUpperBoundRegionSplitPolicy.class.getName());
109    // Now make it so the mock region has a RegionServerService that will
110    // return 'online regions'.
111    RegionServerServices rss = Mockito.mock(RegionServerServices.class);
112    final List<HRegion> regions = new ArrayList<>();
113    Mockito.doReturn(regions).when(rss).getRegions(TABLENAME);
114    Mockito.when(mockRegion.getRegionServerServices()).thenReturn(rss);
115    // Set max size for this 'table'.
116    long maxSplitSize = 1024L;
117    htd.setMaxFileSize(maxSplitSize);
118    // Set flush size to 1/8.  IncreasingToUpperBoundRegionSplitPolicy
119    // grows by the cube of the number of regions times flushsize each time.
120    long flushSize = maxSplitSize/8;
121    conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSize);
122    htd.setMemStoreFlushSize(flushSize);
123    // If RegionServerService with no regions in it -- 'online regions' == 0 --
124    // then IncreasingToUpperBoundRegionSplitPolicy should act like a
125    // ConstantSizePolicy
126    IncreasingToUpperBoundRegionSplitPolicy policy =
127      (IncreasingToUpperBoundRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
128    doConstantSizePolicyTests(policy);
129
130    // Add a store in excess of split size.  Because there are "no regions"
131    // on this server -- rss.getOnlineRegions is 0 -- then we should split
132    // like a constantsizeregionsplitpolicy would
133    HStore mockStore = Mockito.mock(HStore.class);
134    Mockito.doReturn(2000L).when(mockStore).getSize();
135    Mockito.doReturn(true).when(mockStore).canSplit();
136    stores.add(mockStore);
137    // It should split
138    assertTrue(policy.shouldSplit());
139
140    // Now test that we increase our split size as online regions for a table
141    // grows. With one region, split size should be flushsize.
142    regions.add(mockRegion);
143    Mockito.doReturn(flushSize).when(mockStore).getSize();
144    // Should not split since store is flush size.
145    assertFalse(policy.shouldSplit());
146    // Set size of store to be > 2*flush size and we should split
147    Mockito.doReturn(flushSize*2 + 1).when(mockStore).getSize();
148    assertTrue(policy.shouldSplit());
149    // Add another region to the 'online regions' on this server and we should
150    // now be no longer be splittable since split size has gone up.
151    regions.add(mockRegion);
152    assertFalse(policy.shouldSplit());
153    // make sure its just over; verify it'll split
154    Mockito.doReturn((long)(maxSplitSize * 1.25 + 1)).when(mockStore).getSize();
155    assertTrue(policy.shouldSplit());
156
157    // Finally assert that even if loads of regions, we'll split at max size
158    assertWithinJitter(maxSplitSize, policy.getSizeToCheck(1000));
159    // Assert same is true if count of regions is zero.
160    assertWithinJitter(maxSplitSize, policy.getSizeToCheck(0));
161  }
162
163  @Test
164  public void testBusyRegionSplitPolicy() throws Exception {
165    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
166        BusyRegionSplitPolicy.class.getName());
167    conf.setLong("hbase.busy.policy.minAge", 1000000L);
168    conf.setFloat("hbase.busy.policy.blockedRequests", 0.1f);
169
170    RegionServerServices rss  = Mockito.mock(RegionServerServices.class);
171    final List<HRegion> regions = new ArrayList<>();
172    Mockito.doReturn(regions).when(rss).getRegions(TABLENAME);
173    Mockito.when(mockRegion.getRegionServerServices()).thenReturn(rss);
174    Mockito.when(mockRegion.getBlockedRequestsCount()).thenReturn(0L);
175    Mockito.when(mockRegion.getWriteRequestsCount()).thenReturn(0L);
176
177
178    BusyRegionSplitPolicy policy =
179        (BusyRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
180
181    Mockito.when(mockRegion.getBlockedRequestsCount()).thenReturn(10L);
182    Mockito.when(mockRegion.getWriteRequestsCount()).thenReturn(10L);
183    // Not enough time since region came online
184    assertFalse(policy.shouldSplit());
185
186
187    // Reset min age for split to zero
188    conf.setLong("hbase.busy.policy.minAge", 0L);
189    // Aggregate over 500 ms periods
190    conf.setLong("hbase.busy.policy.aggWindow", 500L);
191    policy =
192        (BusyRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
193    long start = EnvironmentEdgeManager.currentTime();
194    Mockito.when(mockRegion.getBlockedRequestsCount()).thenReturn(10L);
195    Mockito.when(mockRegion.getWriteRequestsCount()).thenReturn(20L);
196    Thread.sleep(300);
197    assertFalse(policy.shouldSplit());
198    Mockito.when(mockRegion.getBlockedRequestsCount()).thenReturn(12L);
199    Mockito.when(mockRegion.getWriteRequestsCount()).thenReturn(30L);
200    Thread.sleep(2);
201    // Enough blocked requests since last time, but aggregate blocked request
202    // rate over last 500 ms is still low, because major portion of the window is constituted
203    // by the previous zero blocked request period which lasted at least 300 ms off last 500 ms.
204    if (EnvironmentEdgeManager.currentTime() - start < 500) {
205      assertFalse(policy.shouldSplit());
206    }
207    Mockito.when(mockRegion.getBlockedRequestsCount()).thenReturn(14L);
208    Mockito.when(mockRegion.getWriteRequestsCount()).thenReturn(40L);
209    Thread.sleep(200);
210    assertTrue(policy.shouldSplit());
211  }
212
213  private void assertWithinJitter(long maxSplitSize, long sizeToCheck) {
214    assertTrue("Size greater than lower bound of jitter",
215        (long)(maxSplitSize * 0.75) <= sizeToCheck);
216    assertTrue("Size less than upper bound of jitter",
217        (long)(maxSplitSize * 1.25) >= sizeToCheck);
218  }
219
220  @Test
221  public void testCreateDefault() throws IOException {
222    conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L);
223
224    // Using a default HTD, should pick up the file size from
225    // configuration.
226    ConstantSizeRegionSplitPolicy policy =
227        (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(
228            mockRegion, conf);
229    assertWithinJitter(1234L, policy.getDesiredMaxFileSize());
230
231    // If specified in HTD, should use that
232    htd.setMaxFileSize(9999L);
233    policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(
234        mockRegion, conf);
235    assertWithinJitter(9999L, policy.getDesiredMaxFileSize());
236  }
237
238  /**
239   * Test setting up a customized split policy
240   */
241  @Test
242  public void testCustomPolicy() throws IOException {
243    HTableDescriptor myHtd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
244    myHtd.setValue(HTableDescriptor.SPLIT_POLICY,
245        KeyPrefixRegionSplitPolicy.class.getName());
246    myHtd.setValue(KeyPrefixRegionSplitPolicy.PREFIX_LENGTH_KEY, String.valueOf(2));
247
248    HRegion myMockRegion = Mockito.mock(HRegion.class);
249    Mockito.doReturn(myHtd).when(myMockRegion).getTableDescriptor();
250    Mockito.doReturn(stores).when(myMockRegion).getStores();
251
252    HStore mockStore = Mockito.mock(HStore.class);
253    Mockito.doReturn(2000L).when(mockStore).getSize();
254    Mockito.doReturn(true).when(mockStore).canSplit();
255    Mockito.doReturn(Optional.of(Bytes.toBytes("abcd"))).when(mockStore).getSplitPoint();
256    stores.add(mockStore);
257
258    KeyPrefixRegionSplitPolicy policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy
259        .create(myMockRegion, conf);
260
261    assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
262
263    Mockito.doReturn(true).when(myMockRegion).shouldForceSplit();
264    Mockito.doReturn(Bytes.toBytes("efgh")).when(myMockRegion)
265        .getExplicitSplitPoint();
266
267    policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy
268        .create(myMockRegion, conf);
269
270    assertEquals("ef", Bytes.toString(policy.getSplitPoint()));
271  }
272
273  @Test
274  public void testConstantSizePolicy() throws IOException {
275    htd.setMaxFileSize(1024L);
276    ConstantSizeRegionSplitPolicy policy =
277      (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
278    doConstantSizePolicyTests(policy);
279  }
280
281  /**
282   * Run through tests for a ConstantSizeRegionSplitPolicy
283   * @param policy
284   */
285  private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) {
286    // For no stores, should not split
287    assertFalse(policy.shouldSplit());
288
289    // Add a store above the requisite size. Should split.
290    HStore mockStore = Mockito.mock(HStore.class);
291    Mockito.doReturn(2000L).when(mockStore).getSize();
292    Mockito.doReturn(true).when(mockStore).canSplit();
293    stores.add(mockStore);
294
295    assertTrue(policy.shouldSplit());
296
297    // Act as if there's a reference file or some other reason it can't split.
298    // This should prevent splitting even though it's big enough.
299    Mockito.doReturn(false).when(mockStore).canSplit();
300    assertFalse(policy.shouldSplit());
301
302    // Reset splittability after above
303    Mockito.doReturn(true).when(mockStore).canSplit();
304
305    // Set to a small size but turn on forceSplit. Should result in a split.
306    Mockito.doReturn(true).when(mockRegion).shouldForceSplit();
307    Mockito.doReturn(100L).when(mockStore).getSize();
308    assertTrue(policy.shouldSplit());
309
310    // Turn off forceSplit, should not split
311    Mockito.doReturn(false).when(mockRegion).shouldForceSplit();
312    assertFalse(policy.shouldSplit());
313
314    // Clear families we added above
315    stores.clear();
316  }
317
318  @Test
319  public void testGetSplitPoint() throws IOException {
320    ConstantSizeRegionSplitPolicy policy =
321      (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
322
323    // For no stores, should not split
324    assertFalse(policy.shouldSplit());
325    assertNull(policy.getSplitPoint());
326
327    // Add a store above the requisite size. Should split.
328    HStore mockStore = Mockito.mock(HStore.class);
329    Mockito.doReturn(2000L).when(mockStore).getSize();
330    Mockito.doReturn(true).when(mockStore).canSplit();
331    Mockito.doReturn(Optional.of(Bytes.toBytes("store 1 split"))).when(mockStore).getSplitPoint();
332    stores.add(mockStore);
333
334    assertEquals("store 1 split",
335        Bytes.toString(policy.getSplitPoint()));
336
337    // Add a bigger store. The split point should come from that one
338    HStore mockStore2 = Mockito.mock(HStore.class);
339    Mockito.doReturn(4000L).when(mockStore2).getSize();
340    Mockito.doReturn(true).when(mockStore2).canSplit();
341    Mockito.doReturn(Optional.of(Bytes.toBytes("store 2 split"))).when(mockStore2).getSplitPoint();
342    stores.add(mockStore2);
343
344    assertEquals("store 2 split",
345        Bytes.toString(policy.getSplitPoint()));
346  }
347
348  @Test
349  public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException {
350    HTableDescriptor myHtd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
351    myHtd.setValue(HTableDescriptor.SPLIT_POLICY,
352        DelimitedKeyPrefixRegionSplitPolicy.class.getName());
353    myHtd.setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ",");
354
355    HRegion myMockRegion = Mockito.mock(HRegion.class);
356    Mockito.doReturn(myHtd).when(myMockRegion).getTableDescriptor();
357    Mockito.doReturn(stores).when(myMockRegion).getStores();
358
359    HStore mockStore = Mockito.mock(HStore.class);
360    Mockito.doReturn(2000L).when(mockStore).getSize();
361    Mockito.doReturn(true).when(mockStore).canSplit();
362    Mockito.doReturn(Optional.of(Bytes.toBytes("ab,cd"))).when(mockStore).getSplitPoint();
363    stores.add(mockStore);
364
365    DelimitedKeyPrefixRegionSplitPolicy policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
366        .create(myMockRegion, conf);
367
368    assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
369
370    Mockito.doReturn(true).when(myMockRegion).shouldForceSplit();
371    Mockito.doReturn(Bytes.toBytes("efg,h")).when(myMockRegion)
372        .getExplicitSplitPoint();
373
374    policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
375        .create(myMockRegion, conf);
376
377    assertEquals("efg", Bytes.toString(policy.getSplitPoint()));
378
379    Mockito.doReturn(Bytes.toBytes("ijk")).when(myMockRegion)
380    .getExplicitSplitPoint();
381    assertEquals("ijk", Bytes.toString(policy.getSplitPoint()));
382  }
383
384  @Test
385  public void testConstantSizePolicyWithJitter() throws IOException {
386    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
387      ConstantSizeRegionSplitPolicy.class.getName());
388    htd.setMaxFileSize(Long.MAX_VALUE);
389    boolean positiveJitter = false;
390    ConstantSizeRegionSplitPolicy policy = null;
391    while (!positiveJitter) {
392      policy = (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
393      positiveJitter = policy.positiveJitterRate();
394    }
395    // add a store
396    HStore mockStore = Mockito.mock(HStore.class);
397    Mockito.doReturn(2000L).when(mockStore).getSize();
398    Mockito.doReturn(true).when(mockStore).canSplit();
399    stores.add(mockStore);
400    // Jitter shouldn't cause overflow when HTableDescriptor.MAX_FILESIZE set to Long.MAX_VALUE
401    assertFalse(policy.shouldSplit());
402  }
403
404}