001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.apache.hadoop.hbase.regionserver.StripeStoreFileManager.OPEN_KEY; 021import static org.junit.Assert.assertArrayEquals; 022import static org.junit.Assert.assertEquals; 023import static org.junit.Assert.assertFalse; 024import static org.junit.Assert.assertTrue; 025import static org.junit.Assert.fail; 026 027import java.io.IOException; 028import java.util.ArrayList; 029import java.util.Arrays; 030import java.util.Collection; 031import java.util.Collections; 032import java.util.Iterator; 033import java.util.List; 034import org.apache.hadoop.conf.Configuration; 035import org.apache.hadoop.fs.FileSystem; 036import org.apache.hadoop.fs.Path; 037import org.apache.hadoop.hbase.CellComparatorImpl; 038import org.apache.hadoop.hbase.HBaseClassTestRule; 039import org.apache.hadoop.hbase.HBaseConfiguration; 040import org.apache.hadoop.hbase.HBaseTestingUtility; 041import org.apache.hadoop.hbase.HConstants; 042import org.apache.hadoop.hbase.KeyValue; 043import org.apache.hadoop.hbase.testclassification.RegionServerTests; 044import org.apache.hadoop.hbase.testclassification.SmallTests; 045import org.apache.hadoop.hbase.util.Bytes; 046import org.junit.After; 047import org.junit.Before; 048import org.junit.ClassRule; 049import org.junit.Test; 050import org.junit.experimental.categories.Category; 051import org.mockito.Mockito; 052 053@Category({RegionServerTests.class, SmallTests.class}) 054public class TestStripeStoreFileManager { 055 056 @ClassRule 057 public static final HBaseClassTestRule CLASS_RULE = 058 HBaseClassTestRule.forClass(TestStripeStoreFileManager.class); 059 060 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 061 private static final Path BASEDIR = 062 TEST_UTIL.getDataTestDir(TestStripeStoreFileManager.class.getSimpleName()); 063 private static final Path CFDIR = HStore.getStoreHomedir(BASEDIR, "region", Bytes.toBytes("cf")); 064 065 private static final byte[] KEY_A = Bytes.toBytes("aaa"); 066 private static final byte[] KEY_B = Bytes.toBytes("aab"); 067 private static final byte[] KEY_C = Bytes.toBytes("aac"); 068 private static final byte[] KEY_D = Bytes.toBytes("aad"); 069 070 private static final KeyValue KV_A = new KeyValue(KEY_A, 0L); 071 private static final KeyValue KV_B = new KeyValue(KEY_B, 0L); 072 private static final KeyValue KV_C = new KeyValue(KEY_C, 0L); 073 private static final KeyValue KV_D = new KeyValue(KEY_D, 0L); 074 075 @Before 076 public void setUp() throws Exception { 077 FileSystem fs = TEST_UTIL.getTestFileSystem(); 078 if (!fs.mkdirs(CFDIR)) { 079 throw new IOException("Cannot create test directory " + CFDIR); 080 } 081 } 082 083 @After 084 public void tearDown() throws Exception { 085 FileSystem fs = TEST_UTIL.getTestFileSystem(); 086 if (fs.exists(CFDIR) && !fs.delete(CFDIR, true)) { 087 throw new IOException("Cannot delete test directory " + CFDIR); 088 } 089 } 090 091 @Test 092 public void testInsertFilesIntoL0() throws Exception { 093 StripeStoreFileManager manager = createManager(); 094 MockHStoreFile sf = createFile(); 095 manager.insertNewFiles(al(sf)); 096 assertEquals(1, manager.getStorefileCount()); 097 Collection<HStoreFile> filesForGet = manager.getFilesForScan(KEY_A, true, KEY_A, true); 098 assertEquals(1, filesForGet.size()); 099 assertTrue(filesForGet.contains(sf)); 100 101 // Add some stripes and make sure we get this file for every stripe. 102 manager.addCompactionResults(al(), al(createFile(OPEN_KEY, KEY_B), 103 createFile(KEY_B, OPEN_KEY))); 104 assertTrue(manager.getFilesForScan(KEY_A, true, KEY_A, true).contains(sf)); 105 assertTrue(manager.getFilesForScan(KEY_C, true, KEY_C, true).contains(sf)); 106 } 107 108 @Test 109 public void testClearFiles() throws Exception { 110 StripeStoreFileManager manager = createManager(); 111 manager.insertNewFiles(al(createFile())); 112 manager.insertNewFiles(al(createFile())); 113 manager.addCompactionResults(al(), al(createFile(OPEN_KEY, KEY_B), 114 createFile(KEY_B, OPEN_KEY))); 115 assertEquals(4, manager.getStorefileCount()); 116 Collection<HStoreFile> allFiles = manager.clearFiles(); 117 assertEquals(4, allFiles.size()); 118 assertEquals(0, manager.getStorefileCount()); 119 assertEquals(0, manager.getStorefiles().size()); 120 } 121 122 private static ArrayList<HStoreFile> dumpIterator(Iterator<HStoreFile> iter) { 123 ArrayList<HStoreFile> result = new ArrayList<>(); 124 for (; iter.hasNext(); result.add(iter.next())); 125 return result; 126 } 127 128 @Test 129 public void testRowKeyBefore() throws Exception { 130 StripeStoreFileManager manager = createManager(); 131 HStoreFile l0File = createFile(), l0File2 = createFile(); 132 manager.insertNewFiles(al(l0File)); 133 manager.insertNewFiles(al(l0File2)); 134 // Get candidate files. 135 Iterator<HStoreFile> sfs = manager.getCandidateFilesForRowKeyBefore(KV_B); 136 sfs.next(); 137 sfs.remove(); 138 // Suppose we found a candidate in this file... make sure L0 file remaining is not removed. 139 sfs = manager.updateCandidateFilesForRowKeyBefore(sfs, KV_B, KV_A); 140 assertTrue(sfs.hasNext()); 141 // Now add some stripes (remove L0 file too) 142 MockHStoreFile stripe0a = createFile(0, 100, OPEN_KEY, KEY_B), 143 stripe1 = createFile(KEY_B, OPEN_KEY); 144 manager.addCompactionResults(al(l0File), al(stripe0a, stripe1)); 145 manager.removeCompactedFiles(al(l0File)); 146 // If we want a key <= KEY_A, we should get everything except stripe1. 147 ArrayList<HStoreFile> sfsDump = dumpIterator(manager.getCandidateFilesForRowKeyBefore(KV_A)); 148 assertEquals(2, sfsDump.size()); 149 assertTrue(sfsDump.contains(stripe0a)); 150 assertFalse(sfsDump.contains(stripe1)); 151 // If we want a key <= KEY_B, we should get everything since lower bound is inclusive. 152 sfsDump = dumpIterator(manager.getCandidateFilesForRowKeyBefore(KV_B)); 153 assertEquals(3, sfsDump.size()); 154 assertTrue(sfsDump.contains(stripe1)); 155 // For KEY_D, we should also get everything. 156 sfsDump = dumpIterator(manager.getCandidateFilesForRowKeyBefore(KV_D)); 157 assertEquals(3, sfsDump.size()); 158 // Suppose in the first file we found candidate with KEY_C. 159 // Then, stripe0 no longer matters and should be removed, but stripe1 should stay. 160 sfs = manager.getCandidateFilesForRowKeyBefore(KV_D); 161 sfs.next(); // Skip L0 file. 162 sfs.remove(); 163 sfs = manager.updateCandidateFilesForRowKeyBefore(sfs, KV_D, KV_C); 164 assertEquals(stripe1, sfs.next()); 165 assertFalse(sfs.hasNext()); 166 // Add one more, later, file to stripe0, remove the last annoying L0 file. 167 // This file should be returned in preference to older L0 file; also, after we get 168 // a candidate from the first file, the old one should not be removed. 169 HStoreFile stripe0b = createFile(0, 101, OPEN_KEY, KEY_B); 170 manager.addCompactionResults(al(l0File2), al(stripe0b)); 171 manager.removeCompactedFiles(al(l0File2)); 172 sfs = manager.getCandidateFilesForRowKeyBefore(KV_A); 173 assertEquals(stripe0b, sfs.next()); 174 sfs.remove(); 175 sfs = manager.updateCandidateFilesForRowKeyBefore(sfs, KV_A, KV_A); 176 assertEquals(stripe0a, sfs.next()); 177 } 178 179 @Test 180 public void testGetSplitPointEdgeCases() throws Exception { 181 StripeStoreFileManager manager = createManager(); 182 // No files => no split. 183 assertFalse(manager.getSplitPoint().isPresent()); 184 185 // If there are no stripes, should pick midpoint from the biggest file in L0. 186 MockHStoreFile sf5 = createFile(5, 0); 187 sf5.splitPoint = new byte[] { 1 }; 188 manager.insertNewFiles(al(sf5)); 189 manager.insertNewFiles(al(createFile(1, 0))); 190 assertArrayEquals(sf5.splitPoint, manager.getSplitPoint().get()); 191 192 // Same if there's one stripe but the biggest file is still in L0. 193 manager.addCompactionResults(al(), al(createFile(2, 0, OPEN_KEY, OPEN_KEY))); 194 assertArrayEquals(sf5.splitPoint, manager.getSplitPoint().get()); 195 196 // If the biggest file is in the stripe, should get from it. 197 MockHStoreFile sf6 = createFile(6, 0, OPEN_KEY, OPEN_KEY); 198 sf6.splitPoint = new byte[] { 2 }; 199 manager.addCompactionResults(al(), al(sf6)); 200 assertArrayEquals(sf6.splitPoint, manager.getSplitPoint().get()); 201 } 202 203 @Test 204 public void testGetStripeBoundarySplits() throws Exception { 205 /* First number - split must be after this stripe; further numbers - stripes */ 206 verifySplitPointScenario(5, false, 0f, 2, 1, 1, 1, 1, 1, 10); 207 verifySplitPointScenario(0, false, 0f, 6, 3, 1, 1, 2); 208 verifySplitPointScenario(2, false, 0f, 1, 1, 1, 1, 2); 209 verifySplitPointScenario(0, false, 0f, 5, 4); 210 verifySplitPointScenario(2, false, 0f, 5, 2, 5, 5, 5); 211 } 212 213 @Test 214 public void testGetUnbalancedSplits() throws Exception { 215 /* First number - split must be inside/after this stripe; further numbers - stripes */ 216 verifySplitPointScenario(0, false, 2.1f, 4, 4, 4); // 8/4 is less than 2.1f 217 verifySplitPointScenario(1, true, 1.5f, 4, 4, 4); // 8/4 > 6/6 218 verifySplitPointScenario(1, false, 1.1f, 3, 4, 1, 1, 2, 2); // 7/6 < 8/5 219 verifySplitPointScenario(1, false, 1.1f, 3, 6, 1, 1, 2, 2); // 9/6 == 9/6 220 verifySplitPointScenario(1, true, 1.1f, 3, 8, 1, 1, 2, 2); // 11/6 > 10/7 221 verifySplitPointScenario(3, false, 1.1f, 2, 2, 1, 1, 4, 3); // reverse order 222 verifySplitPointScenario(4, true, 1.1f, 2, 2, 1, 1, 8, 3); // reverse order 223 verifySplitPointScenario(0, true, 1.5f, 10, 4); // 10/4 > 9/5 224 verifySplitPointScenario(0, false, 1.4f, 6, 4); // 6/4 == 6/4 225 verifySplitPointScenario(1, true, 1.5f, 4, 10); // reverse just in case 226 verifySplitPointScenario(0, false, 1.4f, 4, 6); // reverse just in case 227 } 228 229 230 /** 231 * Verifies scenario for finding a split point. 232 * @param splitPointAfter Stripe to expect the split point at/after. 233 * @param shouldSplitStripe If true, the split point is expected in the middle of the above 234 * stripe; if false, should be at the end. 235 * @param splitRatioToVerify Maximum split imbalance ratio. 236 * @param sizes Stripe sizes. 237 */ 238 private void verifySplitPointScenario(int splitPointAfter, boolean shouldSplitStripe, 239 float splitRatioToVerify, int... sizes) throws Exception { 240 assertTrue(sizes.length > 1); 241 ArrayList<HStoreFile> sfs = new ArrayList<>(); 242 for (int sizeIx = 0; sizeIx < sizes.length; ++sizeIx) { 243 byte[] startKey = (sizeIx == 0) ? OPEN_KEY : Bytes.toBytes(sizeIx - 1); 244 byte[] endKey = (sizeIx == sizes.length - 1) ? OPEN_KEY : Bytes.toBytes(sizeIx); 245 MockHStoreFile sf = createFile(sizes[sizeIx], 0, startKey, endKey); 246 sf.splitPoint = Bytes.toBytes(-sizeIx); // set split point to the negative index 247 sfs.add(sf); 248 } 249 250 Configuration conf = HBaseConfiguration.create(); 251 if (splitRatioToVerify != 0) { 252 conf.setFloat(StripeStoreConfig.MAX_REGION_SPLIT_IMBALANCE_KEY, splitRatioToVerify); 253 } 254 StripeStoreFileManager manager = createManager(al(), conf); 255 manager.addCompactionResults(al(), sfs); 256 int result = Bytes.toInt(manager.getSplitPoint().get()); 257 // Either end key and thus positive index, or "middle" of the file and thus negative index. 258 assertEquals(splitPointAfter * (shouldSplitStripe ? -1 : 1), result); 259 } 260 261 private static byte[] keyAfter(byte[] key) { 262 return Arrays.copyOf(key, key.length + 1); 263 } 264 265 @Test 266 public void testGetFilesForGetAndScan() throws Exception { 267 StripeStoreFileManager manager = createManager(); 268 verifyGetAndScanScenario(manager, null, null); 269 verifyGetAndScanScenario(manager, KEY_B, KEY_C); 270 271 // Populate one L0 file. 272 MockHStoreFile sf0 = createFile(); 273 manager.insertNewFiles(al(sf0)); 274 verifyGetAndScanScenario(manager, null, null, sf0); 275 verifyGetAndScanScenario(manager, null, KEY_C, sf0); 276 verifyGetAndScanScenario(manager, KEY_B, null, sf0); 277 verifyGetAndScanScenario(manager, KEY_B, KEY_C, sf0); 278 279 // Populate a bunch of files for stripes, keep L0. 280 MockHStoreFile sfA = createFile(OPEN_KEY, KEY_A); 281 MockHStoreFile sfB = createFile(KEY_A, KEY_B); 282 MockHStoreFile sfC = createFile(KEY_B, KEY_C); 283 MockHStoreFile sfD = createFile(KEY_C, KEY_D); 284 MockHStoreFile sfE = createFile(KEY_D, OPEN_KEY); 285 manager.addCompactionResults(al(), al(sfA, sfB, sfC, sfD, sfE)); 286 287 verifyGetAndScanScenario(manager, null, null, sf0, sfA, sfB, sfC, sfD, sfE); 288 verifyGetAndScanScenario(manager, keyAfter(KEY_A), null, sf0, sfB, sfC, sfD, sfE); 289 verifyGetAndScanScenario(manager, null, keyAfter(KEY_C), sf0, sfA, sfB, sfC, sfD); 290 verifyGetAndScanScenario(manager, KEY_B, null, sf0, sfC, sfD, sfE); 291 verifyGetAndScanScenario(manager, null, KEY_C, sf0, sfA, sfB, sfC, sfD); 292 verifyGetAndScanScenario(manager, KEY_B, keyAfter(KEY_B), sf0, sfC); 293 verifyGetAndScanScenario(manager, keyAfter(KEY_A), KEY_B, sf0, sfB, sfC); 294 verifyGetAndScanScenario(manager, KEY_D, KEY_D, sf0, sfE); 295 verifyGetAndScanScenario(manager, keyAfter(KEY_B), keyAfter(KEY_C), sf0, sfC, sfD); 296 } 297 298 private void verifyGetAndScanScenario(StripeStoreFileManager manager, byte[] start, byte[] end, 299 HStoreFile... results) throws Exception { 300 verifyGetOrScanScenario(manager, start, end, results); 301 } 302 303 @Test 304 @SuppressWarnings("unchecked") 305 public void testLoadFilesWithRecoverableBadFiles() throws Exception { 306 // In L0, there will be file w/o metadata (real L0, 3 files with invalid metadata, and 3 307 // files that overlap valid stripes in various ways). Note that the 4th way to overlap the 308 // stripes will cause the structure to be mostly scraped, and is tested separately. 309 ArrayList<HStoreFile> validStripeFiles = al(createFile(OPEN_KEY, KEY_B), 310 createFile(KEY_B, KEY_C), createFile(KEY_C, OPEN_KEY), 311 createFile(KEY_C, OPEN_KEY)); 312 ArrayList<HStoreFile> filesToGoToL0 = al(createFile(), createFile(null, KEY_A), 313 createFile(KEY_D, null), createFile(KEY_D, KEY_A), createFile(keyAfter(KEY_A), KEY_C), 314 createFile(OPEN_KEY, KEY_D), createFile(KEY_D, keyAfter(KEY_D))); 315 ArrayList<HStoreFile> allFilesToGo = flattenLists(validStripeFiles, filesToGoToL0); 316 Collections.shuffle(allFilesToGo); 317 StripeStoreFileManager manager = createManager(allFilesToGo); 318 List<HStoreFile> l0Files = manager.getLevel0Files(); 319 assertEquals(filesToGoToL0.size(), l0Files.size()); 320 for (HStoreFile sf : filesToGoToL0) { 321 assertTrue(l0Files.contains(sf)); 322 } 323 verifyAllFiles(manager, allFilesToGo); 324 } 325 326 @Test 327 public void testLoadFilesWithBadStripe() throws Exception { 328 // Current "algorithm" will see the after-B key before C key, add it as valid stripe, 329 // and then fail all other stripes. So everything would end up in L0. 330 ArrayList<HStoreFile> allFilesToGo = al(createFile(OPEN_KEY, KEY_B), 331 createFile(KEY_B, KEY_C), createFile(KEY_C, OPEN_KEY), 332 createFile(KEY_B, keyAfter(KEY_B))); 333 Collections.shuffle(allFilesToGo); 334 StripeStoreFileManager manager = createManager(allFilesToGo); 335 assertEquals(allFilesToGo.size(), manager.getLevel0Files().size()); 336 } 337 338 @Test 339 public void testLoadFilesWithGaps() throws Exception { 340 // Stripes must not have gaps. If they do, everything goes to L0. 341 StripeStoreFileManager manager = 342 createManager(al(createFile(OPEN_KEY, KEY_B), createFile(KEY_C, OPEN_KEY))); 343 assertEquals(2, manager.getLevel0Files().size()); 344 // Just one open stripe should be ok. 345 manager = createManager(al(createFile(OPEN_KEY, OPEN_KEY))); 346 assertEquals(0, manager.getLevel0Files().size()); 347 assertEquals(1, manager.getStorefileCount()); 348 } 349 350 @Test 351 public void testLoadFilesAfterSplit() throws Exception { 352 // If stripes are good but have non-open ends, they must be treated as open ends. 353 MockHStoreFile sf = createFile(KEY_B, KEY_C); 354 StripeStoreFileManager manager = createManager(al(createFile(OPEN_KEY, KEY_B), sf)); 355 assertEquals(0, manager.getLevel0Files().size()); 356 // Here, [B, C] is logically [B, inf), so we should be able to compact it to that only. 357 verifyInvalidCompactionScenario(manager, al(sf), al(createFile(KEY_B, KEY_C))); 358 manager.addCompactionResults(al(sf), al(createFile(KEY_B, OPEN_KEY))); 359 manager.removeCompactedFiles(al(sf)); 360 // Do the same for other variants. 361 manager = createManager(al(sf, createFile(KEY_C, OPEN_KEY))); 362 verifyInvalidCompactionScenario(manager, al(sf), al(createFile(KEY_B, KEY_C))); 363 manager.addCompactionResults(al(sf), al(createFile(OPEN_KEY, KEY_C))); 364 manager.removeCompactedFiles(al(sf)); 365 manager = createManager(al(sf)); 366 verifyInvalidCompactionScenario(manager, al(sf), al(createFile(KEY_B, KEY_C))); 367 manager.addCompactionResults(al(sf), al(createFile(OPEN_KEY, OPEN_KEY))); 368 } 369 370 @Test 371 public void testAddingCompactionResults() throws Exception { 372 StripeStoreFileManager manager = createManager(); 373 // First, add some L0 files and "compact" one with new stripe creation. 374 HStoreFile sf_L0_0a = createFile(), sf_L0_0b = createFile(); 375 manager.insertNewFiles(al(sf_L0_0a, sf_L0_0b)); 376 377 // Try compacting with invalid new branches (gaps, overlaps) - no effect. 378 verifyInvalidCompactionScenario(manager, al(sf_L0_0a), al(createFile(OPEN_KEY, KEY_B))); 379 verifyInvalidCompactionScenario(manager, al(sf_L0_0a), al(createFile(OPEN_KEY, KEY_B), 380 createFile(KEY_C, OPEN_KEY))); 381 verifyInvalidCompactionScenario(manager, al(sf_L0_0a), al(createFile(OPEN_KEY, KEY_B), 382 createFile(KEY_B, OPEN_KEY), createFile(KEY_A, KEY_D))); 383 verifyInvalidCompactionScenario(manager, al(sf_L0_0a), al(createFile(OPEN_KEY, KEY_B), 384 createFile(KEY_A, KEY_B), createFile(KEY_B, OPEN_KEY))); 385 386 HStoreFile sf_i2B_0 = createFile(OPEN_KEY, KEY_B); 387 HStoreFile sf_B2C_0 = createFile(KEY_B, KEY_C); 388 HStoreFile sf_C2i_0 = createFile(KEY_C, OPEN_KEY); 389 manager.addCompactionResults(al(sf_L0_0a), al(sf_i2B_0, sf_B2C_0, sf_C2i_0)); 390 manager.removeCompactedFiles(al(sf_L0_0a)); 391 verifyAllFiles(manager, al(sf_L0_0b, sf_i2B_0, sf_B2C_0, sf_C2i_0)); 392 393 // Add another l0 file, "compact" both L0 into two stripes 394 HStoreFile sf_L0_1 = createFile(); 395 HStoreFile sf_i2B_1 = createFile(OPEN_KEY, KEY_B); 396 HStoreFile sf_B2C_1 = createFile(KEY_B, KEY_C); 397 manager.insertNewFiles(al(sf_L0_1)); 398 manager.addCompactionResults(al(sf_L0_0b, sf_L0_1), al(sf_i2B_1, sf_B2C_1)); 399 manager.removeCompactedFiles(al(sf_L0_0b, sf_L0_1)); 400 verifyAllFiles(manager, al(sf_i2B_0, sf_B2C_0, sf_C2i_0, sf_i2B_1, sf_B2C_1)); 401 402 // Try compacting with invalid file (no metadata) - should add files to L0. 403 HStoreFile sf_L0_2 = createFile(null, null); 404 manager.addCompactionResults(al(), al(sf_L0_2)); 405 manager.removeCompactedFiles(al()); 406 verifyAllFiles(manager, al(sf_i2B_0, sf_B2C_0, sf_C2i_0, sf_i2B_1, sf_B2C_1, sf_L0_2)); 407 // Remove it... 408 manager.addCompactionResults(al(sf_L0_2), al()); 409 manager.removeCompactedFiles(al(sf_L0_2)); 410 411 // Do regular compaction in the first stripe. 412 HStoreFile sf_i2B_3 = createFile(OPEN_KEY, KEY_B); 413 manager.addCompactionResults(al(sf_i2B_0, sf_i2B_1), al(sf_i2B_3)); 414 manager.removeCompactedFiles(al(sf_i2B_0, sf_i2B_1)); 415 verifyAllFiles(manager, al(sf_B2C_0, sf_C2i_0, sf_B2C_1, sf_i2B_3)); 416 417 // Rebalance two stripes. 418 HStoreFile sf_B2D_4 = createFile(KEY_B, KEY_D); 419 HStoreFile sf_D2i_4 = createFile(KEY_D, OPEN_KEY); 420 manager.addCompactionResults(al(sf_B2C_0, sf_C2i_0, sf_B2C_1), al(sf_B2D_4, sf_D2i_4)); 421 manager.removeCompactedFiles(al(sf_B2C_0, sf_C2i_0, sf_B2C_1)); 422 verifyAllFiles(manager, al(sf_i2B_3, sf_B2D_4, sf_D2i_4)); 423 424 // Split the first stripe. 425 HStoreFile sf_i2A_5 = createFile(OPEN_KEY, KEY_A); 426 HStoreFile sf_A2B_5 = createFile(KEY_A, KEY_B); 427 manager.addCompactionResults(al(sf_i2B_3), al(sf_i2A_5, sf_A2B_5)); 428 manager.removeCompactedFiles(al(sf_i2B_3)); 429 verifyAllFiles(manager, al(sf_B2D_4, sf_D2i_4, sf_i2A_5, sf_A2B_5)); 430 431 // Split the middle stripe. 432 HStoreFile sf_B2C_6 = createFile(KEY_B, KEY_C); 433 HStoreFile sf_C2D_6 = createFile(KEY_C, KEY_D); 434 manager.addCompactionResults(al(sf_B2D_4), al(sf_B2C_6, sf_C2D_6)); 435 manager.removeCompactedFiles(al(sf_B2D_4)); 436 verifyAllFiles(manager, al(sf_D2i_4, sf_i2A_5, sf_A2B_5, sf_B2C_6, sf_C2D_6)); 437 438 // Merge two different middle stripes. 439 HStoreFile sf_A2C_7 = createFile(KEY_A, KEY_C); 440 manager.addCompactionResults(al(sf_A2B_5, sf_B2C_6), al(sf_A2C_7)); 441 manager.removeCompactedFiles(al(sf_A2B_5, sf_B2C_6)); 442 verifyAllFiles(manager, al(sf_D2i_4, sf_i2A_5, sf_C2D_6, sf_A2C_7)); 443 444 // Merge lower half. 445 HStoreFile sf_i2C_8 = createFile(OPEN_KEY, KEY_C); 446 manager.addCompactionResults(al(sf_i2A_5, sf_A2C_7), al(sf_i2C_8)); 447 manager.removeCompactedFiles(al(sf_i2A_5, sf_A2C_7)); 448 verifyAllFiles(manager, al(sf_D2i_4, sf_C2D_6, sf_i2C_8)); 449 450 // Merge all. 451 HStoreFile sf_i2i_9 = createFile(OPEN_KEY, OPEN_KEY); 452 manager.addCompactionResults(al(sf_D2i_4, sf_C2D_6, sf_i2C_8), al(sf_i2i_9)); 453 manager.removeCompactedFiles(al(sf_D2i_4, sf_C2D_6, sf_i2C_8)); 454 verifyAllFiles(manager, al(sf_i2i_9)); 455 } 456 457 @Test 458 public void testCompactionAndFlushConflict() throws Exception { 459 // Add file flush into stripes 460 StripeStoreFileManager sfm = createManager(); 461 assertEquals(0, sfm.getStripeCount()); 462 HStoreFile sf_i2c = createFile(OPEN_KEY, KEY_C), sf_c2i = createFile(KEY_C, OPEN_KEY); 463 sfm.insertNewFiles(al(sf_i2c, sf_c2i)); 464 assertEquals(2, sfm.getStripeCount()); 465 // Now try to add conflicting flush - should throw. 466 HStoreFile sf_i2d = createFile(OPEN_KEY, KEY_D), sf_d2i = createFile(KEY_D, OPEN_KEY); 467 sfm.insertNewFiles(al(sf_i2d, sf_d2i)); 468 assertEquals(2, sfm.getStripeCount()); 469 assertEquals(2, sfm.getLevel0Files().size()); 470 verifyGetAndScanScenario(sfm, KEY_C, KEY_C, sf_i2d, sf_d2i, sf_c2i); 471 // Remove these files. 472 sfm.addCompactionResults(al(sf_i2d, sf_d2i), al()); 473 sfm.removeCompactedFiles(al(sf_i2d, sf_d2i)); 474 assertEquals(0, sfm.getLevel0Files().size()); 475 // Add another file to stripe; then "rebalance" stripes w/o it - the file, which was 476 // presumably flushed during compaction, should go to L0. 477 HStoreFile sf_i2c_2 = createFile(OPEN_KEY, KEY_C); 478 sfm.insertNewFiles(al(sf_i2c_2)); 479 sfm.addCompactionResults(al(sf_i2c, sf_c2i), al(sf_i2d, sf_d2i)); 480 sfm.removeCompactedFiles(al(sf_i2c, sf_c2i)); 481 assertEquals(1, sfm.getLevel0Files().size()); 482 verifyGetAndScanScenario(sfm, KEY_C, KEY_C, sf_i2d, sf_i2c_2); 483 } 484 485 @Test 486 public void testEmptyResultsForStripes() throws Exception { 487 // Test that we can compact L0 into a subset of stripes. 488 StripeStoreFileManager manager = createManager(); 489 HStoreFile sf0a = createFile(); 490 HStoreFile sf0b = createFile(); 491 manager.insertNewFiles(al(sf0a)); 492 manager.insertNewFiles(al(sf0b)); 493 ArrayList<HStoreFile> compacted = al(createFile(OPEN_KEY, KEY_B), 494 createFile(KEY_B, KEY_C), createFile(KEY_C, OPEN_KEY)); 495 manager.addCompactionResults(al(sf0a), compacted); 496 manager.removeCompactedFiles(al(sf0a)); 497 // Next L0 compaction only produces file for the first and last stripe. 498 ArrayList<HStoreFile> compacted2 = al(createFile(OPEN_KEY, KEY_B), createFile(KEY_C, OPEN_KEY)); 499 manager.addCompactionResults(al(sf0b), compacted2); 500 manager.removeCompactedFiles(al(sf0b)); 501 compacted.addAll(compacted2); 502 verifyAllFiles(manager, compacted); 503 } 504 505 @Test 506 public void testPriority() throws Exception { 507 // Expected priority, file limit, stripe count, files per stripe, l0 files. 508 testPriorityScenario(5, 5, 0, 0, 0); 509 testPriorityScenario(2, 5, 0, 0, 3); 510 testPriorityScenario(4, 25, 5, 1, 0); // example case. 511 testPriorityScenario(3, 25, 5, 1, 1); // L0 files counts for all stripes. 512 testPriorityScenario(3, 25, 5, 2, 0); // file to each stripe - same as one L0 file. 513 testPriorityScenario(2, 25, 5, 4, 0); // 1 is priority user, so 2 is returned. 514 testPriorityScenario(2, 25, 5, 4, 4); // don't return higher than user unless over limit. 515 testPriorityScenario(2, 25, 5, 1, 10); // same. 516 testPriorityScenario(0, 25, 5, 4, 5); // at limit. 517 testPriorityScenario(-5, 25, 5, 6, 0); // over limit! 518 testPriorityScenario(-1, 25, 0, 0, 26); // over limit with just L0 519 } 520 521 private void testPriorityScenario(int expectedPriority, 522 int limit, int stripes, int filesInStripe, int l0Files) throws Exception 523 { 524 final byte[][] keys = { KEY_A, KEY_B, KEY_C, KEY_D }; 525 assertTrue(stripes <= keys.length + 1); 526 Configuration conf = TEST_UTIL.getConfiguration(); 527 conf.setInt("hbase.hstore.blockingStoreFiles", limit); 528 StripeStoreFileManager sfm = createManager(al(), conf); 529 for (int i = 0; i < l0Files; ++i) { 530 sfm.insertNewFiles(al(createFile())); 531 } 532 for (int i = 0; i < filesInStripe; ++i) { 533 ArrayList<HStoreFile> stripe = new ArrayList<>(); 534 for (int j = 0; j < stripes; ++j) { 535 stripe.add(createFile( 536 (j == 0) ? OPEN_KEY : keys[j - 1], (j == stripes - 1) ? OPEN_KEY : keys[j])); 537 } 538 sfm.addCompactionResults(al(), stripe); 539 } 540 assertEquals(expectedPriority, sfm.getStoreCompactionPriority()); 541 } 542 543 private void verifyInvalidCompactionScenario(StripeStoreFileManager manager, 544 ArrayList<HStoreFile> filesToCompact, ArrayList<HStoreFile> filesToInsert) throws Exception { 545 Collection<HStoreFile> allFiles = manager.getStorefiles(); 546 try { 547 manager.addCompactionResults(filesToCompact, filesToInsert); 548 fail("Should have thrown"); 549 } catch (IOException ex) { 550 // Ignore it. 551 } 552 verifyAllFiles(manager, allFiles); // must have the same files. 553 } 554 555 private void verifyGetOrScanScenario(StripeStoreFileManager manager, byte[] start, byte[] end, 556 HStoreFile... results) throws Exception { 557 verifyGetOrScanScenario(manager, start, end, Arrays.asList(results)); 558 } 559 560 private void verifyGetOrScanScenario(StripeStoreFileManager manager, byte[] start, byte[] end, 561 Collection<HStoreFile> results) throws Exception { 562 start = start != null ? start : HConstants.EMPTY_START_ROW; 563 end = end != null ? end : HConstants.EMPTY_END_ROW; 564 Collection<HStoreFile> sfs = manager.getFilesForScan(start, true, end, false); 565 assertEquals(results.size(), sfs.size()); 566 for (HStoreFile result : results) { 567 assertTrue(sfs.contains(result)); 568 } 569 } 570 571 private void verifyAllFiles( 572 StripeStoreFileManager manager, Collection<HStoreFile> results) throws Exception { 573 verifyGetOrScanScenario(manager, null, null, results); 574 } 575 576 // TODO: replace with Mockito? 577 private static MockHStoreFile createFile( 578 long size, long seqNum, byte[] startKey, byte[] endKey) throws Exception { 579 FileSystem fs = TEST_UTIL.getTestFileSystem(); 580 Path testFilePath = StoreFileWriter.getUniqueFile(fs, CFDIR); 581 fs.create(testFilePath).close(); 582 MockHStoreFile sf = new MockHStoreFile(TEST_UTIL, testFilePath, size, 0, false, seqNum); 583 if (startKey != null) { 584 sf.setMetadataValue(StripeStoreFileManager.STRIPE_START_KEY, startKey); 585 } 586 if (endKey != null) { 587 sf.setMetadataValue(StripeStoreFileManager.STRIPE_END_KEY, endKey); 588 } 589 return sf; 590 } 591 592 private static MockHStoreFile createFile(long size, long seqNum) throws Exception { 593 return createFile(size, seqNum, null, null); 594 } 595 596 private static MockHStoreFile createFile(byte[] startKey, byte[] endKey) throws Exception { 597 return createFile(0, 0, startKey, endKey); 598 } 599 600 private static MockHStoreFile createFile() throws Exception { 601 return createFile(null, null); 602 } 603 604 private static StripeStoreFileManager createManager() throws Exception { 605 return createManager(new ArrayList<>()); 606 } 607 608 private static StripeStoreFileManager createManager(ArrayList<HStoreFile> sfs) throws Exception { 609 return createManager(sfs, TEST_UTIL.getConfiguration()); 610 } 611 612 private static StripeStoreFileManager createManager( 613 ArrayList<HStoreFile> sfs, Configuration conf) throws Exception { 614 StripeStoreConfig config = new StripeStoreConfig( 615 conf, Mockito.mock(StoreConfigInformation.class)); 616 StripeStoreFileManager result = new StripeStoreFileManager(CellComparatorImpl.COMPARATOR, conf, 617 config); 618 result.loadFiles(sfs); 619 return result; 620 } 621 622 private static ArrayList<HStoreFile> al(HStoreFile... sfs) { 623 return new ArrayList<>(Arrays.asList(sfs)); 624 } 625 626 private static ArrayList<HStoreFile> flattenLists(ArrayList<HStoreFile>... sfls) { 627 ArrayList<HStoreFile> result = new ArrayList<>(); 628 for (ArrayList<HStoreFile> sfl : sfls) { 629 result.addAll(sfl); 630 } 631 return result; 632 } 633}