001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertNull; 023import static org.junit.Assert.assertTrue; 024import static org.mockito.Mockito.doReturn; 025import static org.mockito.Mockito.mock; 026import static org.mockito.Mockito.when; 027 028import java.io.IOException; 029import java.util.ArrayList; 030import java.util.List; 031import java.util.Optional; 032import org.apache.hadoop.conf.Configuration; 033import org.apache.hadoop.hbase.HBaseClassTestRule; 034import org.apache.hadoop.hbase.HBaseConfiguration; 035import org.apache.hadoop.hbase.HConstants; 036import org.apache.hadoop.hbase.TableName; 037import org.apache.hadoop.hbase.client.RegionInfo; 038import org.apache.hadoop.hbase.client.RegionInfoBuilder; 039import org.apache.hadoop.hbase.client.TableDescriptor; 040import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 041import org.apache.hadoop.hbase.testclassification.RegionServerTests; 042import org.apache.hadoop.hbase.testclassification.SmallTests; 043import org.apache.hadoop.hbase.util.Bytes; 044import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 045import org.junit.Before; 046import org.junit.ClassRule; 047import org.junit.Test; 048import org.junit.experimental.categories.Category; 049 050@Category({ RegionServerTests.class, SmallTests.class }) 051public class TestRegionSplitPolicy { 052 053 @ClassRule 054 public static final HBaseClassTestRule CLASS_RULE = 055 HBaseClassTestRule.forClass(TestRegionSplitPolicy.class); 056 057 private Configuration conf; 058 private HRegion mockRegion; 059 private List<HStore> stores; 060 private static final TableName TABLENAME = TableName.valueOf("t"); 061 062 @Before 063 public void setupMocks() { 064 conf = HBaseConfiguration.create(); 065 RegionInfo hri = RegionInfoBuilder.newBuilder(TABLENAME).build(); 066 mockRegion = mock(HRegion.class); 067 doReturn(hri).when(mockRegion).getRegionInfo(); 068 doReturn(true).when(mockRegion).isAvailable(); 069 stores = new ArrayList<>(); 070 doReturn(stores).when(mockRegion).getStores(); 071 } 072 073 @Test 074 public void testForceSplitRegionWithReference() throws IOException { 075 TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(1024L).build(); 076 doReturn(td).when(mockRegion).getTableDescriptor(); 077 // Add a store above the requisite size. Should split. 078 HStore mockStore = mock(HStore.class); 079 doReturn(2000L).when(mockStore).getSize(); 080 // Act as if there's a reference file or some other reason it can't split. 081 // This should prevent splitting even though it's big enough. 082 doReturn(false).when(mockStore).canSplit(); 083 stores.add(mockStore); 084 085 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, 086 ConstantSizeRegionSplitPolicy.class.getName()); 087 ConstantSizeRegionSplitPolicy policy = 088 (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); 089 assertFalse(policy.shouldSplit()); 090 091 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, 092 IncreasingToUpperBoundRegionSplitPolicy.class.getName()); 093 policy = (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); 094 assertFalse(policy.shouldSplit()); 095 } 096 097 @Test 098 public void testIncreasingToUpperBoundRegionSplitPolicy() throws IOException { 099 // Configure IncreasingToUpperBoundRegionSplitPolicy as our split policy 100 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, 101 IncreasingToUpperBoundRegionSplitPolicy.class.getName()); 102 // Now make it so the mock region has a RegionServerService that will 103 // return 'online regions'. 104 RegionServerServices rss = mock(RegionServerServices.class); 105 final List<HRegion> regions = new ArrayList<>(); 106 doReturn(regions).when(rss).getRegions(TABLENAME); 107 when(mockRegion.getRegionServerServices()).thenReturn(rss); 108 // Set max size for this 'table'. 109 long maxSplitSize = 1024L; 110 // Set flush size to 1/8. IncreasingToUpperBoundRegionSplitPolicy 111 // grows by the cube of the number of regions times flushsize each time. 112 long flushSize = maxSplitSize / 8; 113 conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSize); 114 TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(maxSplitSize) 115 .setMemStoreFlushSize(flushSize).build(); 116 doReturn(td).when(mockRegion).getTableDescriptor(); 117 // If RegionServerService with no regions in it -- 'online regions' == 0 -- 118 // then IncreasingToUpperBoundRegionSplitPolicy should act like a 119 // ConstantSizePolicy 120 IncreasingToUpperBoundRegionSplitPolicy policy = 121 (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); 122 doConstantSizePolicyTests(policy); 123 124 // Add a store in excess of split size. Because there are "no regions" 125 // on this server -- rss.getOnlineRegions is 0 -- then we should split 126 // like a constantsizeregionsplitpolicy would 127 HStore mockStore = mock(HStore.class); 128 doReturn(2000L).when(mockStore).getSize(); 129 doReturn(true).when(mockStore).canSplit(); 130 stores.add(mockStore); 131 // It should split 132 assertTrue(policy.shouldSplit()); 133 134 // Now test that we increase our split size as online regions for a table 135 // grows. With one region, split size should be flushsize. 136 regions.add(mockRegion); 137 doReturn(flushSize).when(mockStore).getSize(); 138 // Should not split since store is flush size. 139 assertFalse(policy.shouldSplit()); 140 // Set size of store to be > 2*flush size and we should split 141 doReturn(flushSize * 2 + 1).when(mockStore).getSize(); 142 assertTrue(policy.shouldSplit()); 143 // Add another region to the 'online regions' on this server and we should 144 // now be no longer be splittable since split size has gone up. 145 regions.add(mockRegion); 146 assertFalse(policy.shouldSplit()); 147 // make sure its just over; verify it'll split 148 doReturn((long) (maxSplitSize * 1.25 + 1)).when(mockStore).getSize(); 149 assertTrue(policy.shouldSplit()); 150 151 // Finally assert that even if loads of regions, we'll split at max size 152 assertWithinJitter(maxSplitSize, policy.getSizeToCheck(1000)); 153 // Assert same is true if count of regions is zero. 154 assertWithinJitter(maxSplitSize, policy.getSizeToCheck(0)); 155 } 156 157 @Test 158 public void testIsExceedSize() throws IOException { 159 // Configure SteppingAllStoresSizeSplitPolicy as our split policy 160 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, 161 ConstantSizeRegionSplitPolicy.class.getName()); 162 // Now make it so the mock region has a RegionServerService that will 163 // return 'online regions'. 164 RegionServerServices rss = mock(RegionServerServices.class); 165 final List<HRegion> regions = new ArrayList<>(); 166 doReturn(regions).when(rss).getRegions(TABLENAME); 167 when(mockRegion.getRegionServerServices()).thenReturn(rss); 168 169 TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).build(); 170 doReturn(td).when(mockRegion).getTableDescriptor(); 171 ConstantSizeRegionSplitPolicy policy = 172 (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); 173 regions.add(mockRegion); 174 175 HStore mockStore1 = mock(HStore.class); 176 doReturn(100L).when(mockStore1).getSize(); 177 HStore mockStore2 = mock(HStore.class); 178 doReturn(924L).when(mockStore2).getSize(); 179 HStore mockStore3 = mock(HStore.class); 180 doReturn(925L).when(mockStore3).getSize(); 181 182 // test sum of store's size not greater than sizeToCheck 183 stores.add(mockStore1); 184 stores.add(mockStore2); 185 assertFalse(policy.isExceedSize(1024)); 186 stores.clear(); 187 188 // test sum of store's size greater than sizeToCheck 189 stores.add(mockStore1); 190 stores.add(mockStore3); 191 assertTrue(policy.isExceedSize(1024)); 192 } 193 194 @Test 195 public void testBusyRegionSplitPolicy() throws Exception { 196 doReturn(TableDescriptorBuilder.newBuilder(TABLENAME).build()).when(mockRegion) 197 .getTableDescriptor(); 198 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, BusyRegionSplitPolicy.class.getName()); 199 conf.setLong("hbase.busy.policy.minAge", 1000000L); 200 conf.setFloat("hbase.busy.policy.blockedRequests", 0.1f); 201 202 RegionServerServices rss = mock(RegionServerServices.class); 203 final List<HRegion> regions = new ArrayList<>(); 204 doReturn(regions).when(rss).getRegions(TABLENAME); 205 when(mockRegion.getRegionServerServices()).thenReturn(rss); 206 when(mockRegion.getBlockedRequestsCount()).thenReturn(0L); 207 when(mockRegion.getWriteRequestsCount()).thenReturn(0L); 208 209 BusyRegionSplitPolicy policy = 210 (BusyRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); 211 212 when(mockRegion.getBlockedRequestsCount()).thenReturn(10L); 213 when(mockRegion.getWriteRequestsCount()).thenReturn(10L); 214 // Not enough time since region came online 215 assertFalse(policy.shouldSplit()); 216 217 // Reset min age for split to zero 218 conf.setLong("hbase.busy.policy.minAge", 0L); 219 // Aggregate over 500 ms periods 220 conf.setLong("hbase.busy.policy.aggWindow", 500L); 221 policy = (BusyRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); 222 long start = EnvironmentEdgeManager.currentTime(); 223 when(mockRegion.getBlockedRequestsCount()).thenReturn(10L); 224 when(mockRegion.getWriteRequestsCount()).thenReturn(20L); 225 Thread.sleep(300); 226 assertFalse(policy.shouldSplit()); 227 when(mockRegion.getBlockedRequestsCount()).thenReturn(12L); 228 when(mockRegion.getWriteRequestsCount()).thenReturn(30L); 229 Thread.sleep(2); 230 // Enough blocked requests since last time, but aggregate blocked request 231 // rate over last 500 ms is still low, because major portion of the window is constituted 232 // by the previous zero blocked request period which lasted at least 300 ms off last 500 ms. 233 if (EnvironmentEdgeManager.currentTime() - start < 500) { 234 assertFalse(policy.shouldSplit()); 235 } 236 when(mockRegion.getBlockedRequestsCount()).thenReturn(14L); 237 when(mockRegion.getWriteRequestsCount()).thenReturn(40L); 238 Thread.sleep(200); 239 assertTrue(policy.shouldSplit()); 240 } 241 242 private void assertWithinJitter(long maxSplitSize, long sizeToCheck) { 243 assertTrue("Size greater than lower bound of jitter", 244 (long) (maxSplitSize * 0.75) <= sizeToCheck); 245 assertTrue("Size less than upper bound of jitter", (long) (maxSplitSize * 1.25) >= sizeToCheck); 246 } 247 248 @Test 249 public void testCreateDefault() throws IOException { 250 conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L); 251 TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).build(); 252 doReturn(td).when(mockRegion).getTableDescriptor(); 253 // Using a default HTD, should pick up the file size from 254 // configuration. 255 ConstantSizeRegionSplitPolicy policy = 256 (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); 257 assertWithinJitter(1234L, policy.getDesiredMaxFileSize()); 258 259 // If specified in HTD, should use that 260 td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(9999L).build(); 261 doReturn(td).when(mockRegion).getTableDescriptor(); 262 policy = (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); 263 assertWithinJitter(9999L, policy.getDesiredMaxFileSize()); 264 } 265 266 /** 267 * Test setting up a customized split policy 268 */ 269 @Test 270 public void testCustomPolicy() throws IOException { 271 TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME) 272 .setRegionSplitPolicyClassName(KeyPrefixRegionSplitPolicy.class.getName()) 273 .setValue(KeyPrefixRegionSplitPolicy.PREFIX_LENGTH_KEY, "2").build(); 274 275 doReturn(td).when(mockRegion).getTableDescriptor(); 276 277 HStore mockStore = mock(HStore.class); 278 doReturn(2000L).when(mockStore).getSize(); 279 doReturn(true).when(mockStore).canSplit(); 280 doReturn(Optional.of(Bytes.toBytes("abcd"))).when(mockStore).getSplitPoint(); 281 stores.add(mockStore); 282 283 KeyPrefixRegionSplitPolicy policy = 284 (KeyPrefixRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); 285 286 assertEquals("ab", Bytes.toString(policy.getSplitPoint())); 287 } 288 289 @Test 290 public void testConstantSizePolicy() throws IOException { 291 TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(1024L).build(); 292 doReturn(td).when(mockRegion).getTableDescriptor(); 293 ConstantSizeRegionSplitPolicy policy = 294 (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); 295 doConstantSizePolicyTests(policy); 296 } 297 298 /** 299 * Run through tests for a ConstantSizeRegionSplitPolicy 300 */ 301 private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) { 302 // For no stores, should not split 303 assertFalse(policy.shouldSplit()); 304 305 // Add a store above the requisite size. Should split. 306 HStore mockStore = mock(HStore.class); 307 doReturn(2000L).when(mockStore).getSize(); 308 doReturn(true).when(mockStore).canSplit(); 309 stores.add(mockStore); 310 311 assertTrue(policy.shouldSplit()); 312 313 // Act as if there's a reference file or some other reason it can't split. 314 // This should prevent splitting even though it's big enough. 315 doReturn(false).when(mockStore).canSplit(); 316 assertFalse(policy.shouldSplit()); 317 318 // Reset splittability after above 319 doReturn(true).when(mockStore).canSplit(); 320 321 // Set to a small size, should not split 322 doReturn(100L).when(mockStore).getSize(); 323 assertFalse(policy.shouldSplit()); 324 325 // Clear families we added above 326 stores.clear(); 327 } 328 329 @Test 330 public void testGetSplitPoint() throws IOException { 331 TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).build(); 332 doReturn(td).when(mockRegion).getTableDescriptor(); 333 334 ConstantSizeRegionSplitPolicy policy = 335 (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); 336 337 // For no stores, should not split 338 assertFalse(policy.shouldSplit()); 339 assertNull(policy.getSplitPoint()); 340 341 // Add a store above the requisite size. Should split. 342 HStore mockStore = mock(HStore.class); 343 doReturn(2000L).when(mockStore).getSize(); 344 doReturn(true).when(mockStore).canSplit(); 345 doReturn(Optional.of(Bytes.toBytes("store 1 split"))).when(mockStore).getSplitPoint(); 346 stores.add(mockStore); 347 348 assertEquals("store 1 split", Bytes.toString(policy.getSplitPoint())); 349 350 // Add a bigger store. The split point should come from that one 351 HStore mockStore2 = mock(HStore.class); 352 doReturn(4000L).when(mockStore2).getSize(); 353 doReturn(true).when(mockStore2).canSplit(); 354 doReturn(Optional.of(Bytes.toBytes("store 2 split"))).when(mockStore2).getSplitPoint(); 355 stores.add(mockStore2); 356 357 assertEquals("store 2 split", Bytes.toString(policy.getSplitPoint())); 358 } 359 360 @Test 361 public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException { 362 TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME) 363 .setRegionSplitPolicyClassName(DelimitedKeyPrefixRegionSplitPolicy.class.getName()) 364 .setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ",").build(); 365 366 doReturn(td).when(mockRegion).getTableDescriptor(); 367 doReturn(stores).when(mockRegion).getStores(); 368 369 HStore mockStore = mock(HStore.class); 370 doReturn(2000L).when(mockStore).getSize(); 371 doReturn(true).when(mockStore).canSplit(); 372 doReturn(Optional.of(Bytes.toBytes("ab,cd"))).when(mockStore).getSplitPoint(); 373 stores.add(mockStore); 374 375 DelimitedKeyPrefixRegionSplitPolicy policy = 376 (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); 377 378 assertEquals("ab", Bytes.toString(policy.getSplitPoint())); 379 380 doReturn(Optional.of(Bytes.toBytes("ijk"))).when(mockStore).getSplitPoint(); 381 assertEquals("ijk", Bytes.toString(policy.getSplitPoint())); 382 } 383 384 @Test 385 public void testConstantSizePolicyWithJitter() throws IOException { 386 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, 387 ConstantSizeRegionSplitPolicy.class.getName()); 388 TableDescriptor td = 389 TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(Long.MAX_VALUE).build(); 390 doReturn(td).when(mockRegion).getTableDescriptor(); 391 boolean positiveJitter = false; 392 ConstantSizeRegionSplitPolicy policy = null; 393 while (!positiveJitter) { 394 policy = (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf); 395 positiveJitter = policy.positiveJitterRate(); 396 } 397 // add a store 398 HStore mockStore = mock(HStore.class); 399 doReturn(2000L).when(mockStore).getSize(); 400 doReturn(true).when(mockStore).canSplit(); 401 stores.add(mockStore); 402 // Jitter shouldn't cause overflow when HTableDescriptor.MAX_FILESIZE set to Long.MAX_VALUE 403 assertFalse(policy.shouldSplit()); 404 } 405}