001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.master.janitor; 019 020import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal; 021import static org.junit.Assert.assertEquals; 022import static org.junit.Assert.assertFalse; 023import static org.junit.Assert.assertTrue; 024import static org.mockito.Mockito.any; 025import static org.mockito.Mockito.doAnswer; 026import static org.mockito.Mockito.doReturn; 027import static org.mockito.Mockito.doThrow; 028import static org.mockito.Mockito.spy; 029import static org.mockito.Mockito.when; 030 031import java.io.IOException; 032import java.util.ArrayList; 033import java.util.List; 034import java.util.Map; 035import java.util.NavigableMap; 036import java.util.Objects; 037import java.util.SortedMap; 038import java.util.SortedSet; 039import java.util.TreeMap; 040import java.util.concurrent.ConcurrentSkipListMap; 041import java.util.concurrent.atomic.AtomicBoolean; 042import org.apache.hadoop.fs.FSDataOutputStream; 043import org.apache.hadoop.fs.FileStatus; 044import org.apache.hadoop.fs.FileSystem; 045import org.apache.hadoop.fs.Path; 046import org.apache.hadoop.hbase.HBaseClassTestRule; 047import org.apache.hadoop.hbase.HBaseTestingUtil; 048import org.apache.hadoop.hbase.HConstants; 049import org.apache.hadoop.hbase.MetaMockingUtil; 050import org.apache.hadoop.hbase.ServerName; 051import org.apache.hadoop.hbase.TableName; 052import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 053import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 054import org.apache.hadoop.hbase.client.RegionInfo; 055import org.apache.hadoop.hbase.client.RegionInfoBuilder; 056import org.apache.hadoop.hbase.client.Result; 057import org.apache.hadoop.hbase.client.TableDescriptor; 058import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 059import org.apache.hadoop.hbase.io.Reference; 060import org.apache.hadoop.hbase.master.MasterFileSystem; 061import org.apache.hadoop.hbase.master.MasterServices; 062import org.apache.hadoop.hbase.master.assignment.MockMasterServices; 063import org.apache.hadoop.hbase.master.janitor.CatalogJanitor.SplitParentFirstComparator; 064import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; 065import org.apache.hadoop.hbase.regionserver.ChunkCreator; 066import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; 067import org.apache.hadoop.hbase.regionserver.MemStoreLAB; 068import org.apache.hadoop.hbase.testclassification.MasterTests; 069import org.apache.hadoop.hbase.testclassification.MediumTests; 070import org.apache.hadoop.hbase.util.Bytes; 071import org.apache.hadoop.hbase.util.CommonFSUtils; 072import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 073import org.apache.hadoop.hbase.util.HFileArchiveUtil; 074import org.apache.zookeeper.KeeperException; 075import org.junit.After; 076import org.junit.Before; 077import org.junit.BeforeClass; 078import org.junit.ClassRule; 079import org.junit.Rule; 080import org.junit.Test; 081import org.junit.experimental.categories.Category; 082import org.junit.rules.TestName; 083import org.slf4j.Logger; 084import org.slf4j.LoggerFactory; 085 086@Category({ MasterTests.class, MediumTests.class }) 087public class TestCatalogJanitor { 088 089 @ClassRule 090 public static final HBaseClassTestRule CLASS_RULE = 091 HBaseClassTestRule.forClass(TestCatalogJanitor.class); 092 093 private static final Logger LOG = LoggerFactory.getLogger(TestCatalogJanitor.class); 094 095 private static final HBaseTestingUtil HTU = new HBaseTestingUtil(); 096 097 @Rule 098 public final TestName name = new TestName(); 099 100 private MockMasterServices masterServices; 101 private CatalogJanitor janitor; 102 103 @BeforeClass 104 public static void beforeClass() throws Exception { 105 ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, 106 MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); 107 } 108 109 @Before 110 public void setup() throws IOException, KeeperException { 111 setRootDirAndCleanIt(HTU, this.name.getMethodName()); 112 NavigableMap<ServerName, SortedSet<byte[]>> regionsToRegionServers = 113 new ConcurrentSkipListMap<ServerName, SortedSet<byte[]>>(); 114 this.masterServices = new MockMasterServices(HTU.getConfiguration(), regionsToRegionServers); 115 this.masterServices.start(10, null); 116 this.janitor = new CatalogJanitor(masterServices); 117 } 118 119 @After 120 public void teardown() { 121 this.janitor.shutdown(true); 122 this.masterServices.stop("DONE"); 123 } 124 125 private RegionInfo createRegionInfo(TableName tableName, byte[] startKey, byte[] endKey) { 126 return createRegionInfo(tableName, startKey, endKey, false); 127 } 128 129 private RegionInfo createRegionInfo(TableName tableName, byte[] startKey, byte[] endKey, 130 boolean split) { 131 return RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(endKey) 132 .setSplit(split).build(); 133 } 134 135 @Test 136 public void testCleanMerge() throws IOException { 137 TableDescriptor td = createTableDescriptorForCurrentMethod(); 138 // Create regions. 139 RegionInfo merged = 140 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee")); 141 RegionInfo parenta = 142 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc")); 143 RegionInfo parentb = 144 createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee")); 145 146 List<RegionInfo> parents = new ArrayList<>(); 147 parents.add(parenta); 148 parents.add(parentb); 149 150 Path rootdir = this.masterServices.getMasterFileSystem().getRootDir(); 151 Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName()); 152 Path storedir = 153 HRegionFileSystem.getStoreHomedir(tabledir, merged, td.getColumnFamilies()[0].getName()); 154 155 Path parentaRef = createMergeReferenceFile(storedir, merged, parenta); 156 Path parentbRef = createMergeReferenceFile(storedir, merged, parentb); 157 158 // references exist, should not clean 159 assertFalse(CatalogJanitor.cleanMergeRegion(masterServices, merged, parents)); 160 161 masterServices.getMasterFileSystem().getFileSystem().delete(parentaRef, false); 162 163 // one reference still exists, should not clean 164 assertFalse(CatalogJanitor.cleanMergeRegion(masterServices, merged, parents)); 165 166 masterServices.getMasterFileSystem().getFileSystem().delete(parentbRef, false); 167 168 // all references removed, should clean 169 assertTrue(CatalogJanitor.cleanMergeRegion(masterServices, merged, parents)); 170 } 171 172 @Test 173 public void testDontCleanMergeIfFileSystemException() throws IOException { 174 TableDescriptor td = createTableDescriptorForCurrentMethod(); 175 // Create regions. 176 RegionInfo merged = 177 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee")); 178 RegionInfo parenta = 179 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc")); 180 RegionInfo parentb = 181 createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee")); 182 183 List<RegionInfo> parents = new ArrayList<>(); 184 parents.add(parenta); 185 parents.add(parentb); 186 187 Path rootdir = this.masterServices.getMasterFileSystem().getRootDir(); 188 Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName()); 189 Path storedir = 190 HRegionFileSystem.getStoreHomedir(tabledir, merged, td.getColumnFamilies()[0].getName()); 191 createMergeReferenceFile(storedir, merged, parenta); 192 193 MasterServices mockedMasterServices = spy(masterServices); 194 MasterFileSystem mockedMasterFileSystem = spy(masterServices.getMasterFileSystem()); 195 FileSystem mockedFileSystem = spy(masterServices.getMasterFileSystem().getFileSystem()); 196 197 when(mockedMasterServices.getMasterFileSystem()).thenReturn(mockedMasterFileSystem); 198 when(mockedMasterFileSystem.getFileSystem()).thenReturn(mockedFileSystem); 199 200 // throw on the first exists check 201 doThrow(new IOException("Some exception")).when(mockedFileSystem).exists(any()); 202 203 assertFalse(CatalogJanitor.cleanMergeRegion(mockedMasterServices, merged, parents)); 204 205 // throw on the second exists check (within HRegionfileSystem) 206 AtomicBoolean returned = new AtomicBoolean(false); 207 doAnswer(invocationOnMock -> { 208 if (!returned.get()) { 209 returned.set(true); 210 return masterServices.getMasterFileSystem().getFileSystem() 211 .exists(invocationOnMock.getArgument(0)); 212 } 213 throw new IOException("Some exception"); 214 }).when(mockedFileSystem).exists(any()); 215 216 assertFalse(CatalogJanitor.cleanMergeRegion(mockedMasterServices, merged, parents)); 217 } 218 219 private Path createMergeReferenceFile(Path storeDir, RegionInfo mergedRegion, 220 RegionInfo parentRegion) throws IOException { 221 Reference ref = Reference.createTopReference(mergedRegion.getStartKey()); 222 long now = EnvironmentEdgeManager.currentTime(); 223 // Reference name has this format: StoreFile#REF_NAME_PARSER 224 Path p = new Path(storeDir, Long.toString(now) + "." + parentRegion.getEncodedName()); 225 FileSystem fs = this.masterServices.getMasterFileSystem().getFileSystem(); 226 return ref.write(fs, p); 227 } 228 229 /** 230 * Test clearing a split parent. 231 */ 232 @Test 233 public void testCleanParent() throws IOException, InterruptedException { 234 TableDescriptor td = createTableDescriptorForCurrentMethod(); 235 // Create regions. 236 RegionInfo parent = 237 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee")); 238 RegionInfo splita = 239 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc")); 240 RegionInfo splitb = 241 createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee")); 242 // Test that when both daughter regions are in place, that we do not remove the parent. 243 Result r = createResult(parent, splita, splitb); 244 // Add a reference under splitA directory so we don't clear out the parent. 245 Path rootdir = this.masterServices.getMasterFileSystem().getRootDir(); 246 Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName()); 247 Path parentdir = new Path(tabledir, parent.getEncodedName()); 248 Path storedir = 249 HRegionFileSystem.getStoreHomedir(tabledir, splita, td.getColumnFamilies()[0].getName()); 250 Reference ref = Reference.createTopReference(Bytes.toBytes("ccc")); 251 long now = EnvironmentEdgeManager.currentTime(); 252 // Reference name has this format: StoreFile#REF_NAME_PARSER 253 Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName()); 254 FileSystem fs = this.masterServices.getMasterFileSystem().getFileSystem(); 255 Path path = ref.write(fs, p); 256 assertTrue(fs.exists(path)); 257 LOG.info("Created reference " + path); 258 // Add a parentdir for kicks so can check it gets removed by the catalogjanitor. 259 fs.mkdirs(parentdir); 260 assertFalse(CatalogJanitor.cleanParent(masterServices, parent, r)); 261 ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor()); 262 assertTrue(fs.exists(parentdir)); 263 // Remove the reference file and try again. 264 assertTrue(fs.delete(p, true)); 265 assertTrue(CatalogJanitor.cleanParent(masterServices, parent, r)); 266 // Parent cleanup is run async as a procedure. Make sure parentdir is removed. 267 ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor()); 268 assertTrue(!fs.exists(parentdir)); 269 } 270 271 /** 272 * Make sure parent gets cleaned up even if daughter is cleaned up before it. 273 */ 274 @Test 275 public void testParentCleanedEvenIfDaughterGoneFirst() throws IOException, InterruptedException { 276 parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(this.name.getMethodName(), 277 Bytes.toBytes("eee")); 278 } 279 280 /** 281 * Make sure last parent with empty end key gets cleaned up even if daughter is cleaned up before 282 * it. 283 */ 284 @Test 285 public void testLastParentCleanedEvenIfDaughterGoneFirst() 286 throws IOException, InterruptedException { 287 parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(this.name.getMethodName(), new byte[0]); 288 } 289 290 /** 291 * @return A TableDescriptor with a tableName of current method name and a column family that is 292 * MockMasterServices.DEFAULT_COLUMN_FAMILY_NAME) 293 */ 294 private TableDescriptor createTableDescriptorForCurrentMethod() { 295 ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder 296 .newBuilder(Bytes.toBytes(MockMasterServices.DEFAULT_COLUMN_FAMILY_NAME)).build(); 297 return TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name.getMethodName())) 298 .setColumnFamily(columnFamilyDescriptor).build(); 299 } 300 301 /** 302 * Make sure parent with specified end key gets cleaned up even if daughter is cleaned up before 303 * it. 304 * @param rootDir the test case name, used as the HBase testing utility root 305 * @param lastEndKey the end key of the split parent 306 */ 307 private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(final String rootDir, 308 final byte[] lastEndKey) throws IOException, InterruptedException { 309 TableDescriptor td = createTableDescriptorForCurrentMethod(); 310 // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc. 311 RegionInfo parent = createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), lastEndKey); 312 // Sleep a second else the encoded name on these regions comes out 313 // same for all with same start key and made in same second. 314 Thread.sleep(1001); 315 316 // Daughter a 317 RegionInfo splita = 318 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc")); 319 Thread.sleep(1001); 320 // Make daughters of daughter a; splitaa and splitab. 321 RegionInfo splitaa = 322 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb")); 323 RegionInfo splitab = 324 createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc")); 325 326 // Daughter b 327 RegionInfo splitb = createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), lastEndKey); 328 Thread.sleep(1001); 329 // Make Daughters of daughterb; splitba and splitbb. 330 RegionInfo splitba = 331 createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("ddd")); 332 RegionInfo splitbb = createRegionInfo(td.getTableName(), Bytes.toBytes("ddd"), lastEndKey); 333 334 // First test that our Comparator works right up in CatalogJanitor. 335 SortedMap<RegionInfo, Result> regions = 336 new TreeMap<>(new CatalogJanitor.SplitParentFirstComparator()); 337 // Now make sure that this regions map sorts as we expect it to. 338 regions.put(parent, createResult(parent, splita, splitb)); 339 regions.put(splitb, createResult(splitb, splitba, splitbb)); 340 regions.put(splita, createResult(splita, splitaa, splitab)); 341 // Assert its properly sorted. 342 int index = 0; 343 for (Map.Entry<RegionInfo, Result> e : regions.entrySet()) { 344 if (index == 0) { 345 assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName())); 346 } else if (index == 1) { 347 assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName())); 348 } else if (index == 2) { 349 assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName())); 350 } 351 index++; 352 } 353 354 // Now play around with the cleanParent function. Create a ref from splita up to the parent. 355 Path splitaRef = 356 createReferences(this.masterServices, td, parent, splita, Bytes.toBytes("ccc"), false); 357 // Make sure actual super parent sticks around because splita has a ref. 358 assertFalse(CatalogJanitor.cleanParent(masterServices, parent, regions.get(parent))); 359 360 // splitba, and split bb, do not have dirs in fs. That means that if 361 // we test splitb, it should get cleaned up. 362 assertTrue(CatalogJanitor.cleanParent(masterServices, splitb, regions.get(splitb))); 363 364 // Now remove ref from splita to parent... so parent can be let go and so 365 // the daughter splita can be split (can't split if still references). 366 // BUT make the timing such that the daughter gets cleaned up before we 367 // can get a chance to let go of the parent. 368 FileSystem fs = FileSystem.get(HTU.getConfiguration()); 369 assertTrue(fs.delete(splitaRef, true)); 370 // Create the refs from daughters of splita. 371 Path splitaaRef = 372 createReferences(this.masterServices, td, splita, splitaa, Bytes.toBytes("bbb"), false); 373 Path splitabRef = 374 createReferences(this.masterServices, td, splita, splitab, Bytes.toBytes("bbb"), true); 375 376 // Test splita. It should stick around because references from splitab, etc. 377 assertFalse(CatalogJanitor.cleanParent(masterServices, splita, regions.get(splita))); 378 379 // Now clean up parent daughter first. Remove references from its daughters. 380 assertTrue(fs.delete(splitaaRef, true)); 381 assertTrue(fs.delete(splitabRef, true)); 382 assertTrue(CatalogJanitor.cleanParent(masterServices, splita, regions.get(splita))); 383 384 // Super parent should get cleaned up now both splita and splitb are gone. 385 assertTrue(CatalogJanitor.cleanParent(masterServices, parent, regions.get(parent))); 386 } 387 388 /** 389 * CatalogJanitor.scan() should not clean parent regions if their own parents are still 390 * referencing them. This ensures that grandparent regions do not point to deleted parent regions. 391 */ 392 @Test 393 public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception { 394 TableDescriptor td = createTableDescriptorForCurrentMethod(); 395 // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc. 396 397 // Parent 398 RegionInfo parent = 399 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), HConstants.EMPTY_BYTE_ARRAY, true); 400 // Sleep a second else the encoded name on these regions comes out 401 // same for all with same start key and made in same second. 402 Thread.sleep(1001); 403 404 // Daughter a 405 RegionInfo splita = 406 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"), true); 407 Thread.sleep(1001); 408 409 // Make daughters of daughter a; splitaa and splitab. 410 RegionInfo splitaa = 411 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), false); 412 RegionInfo splitab = 413 createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false); 414 415 // Daughter b 416 RegionInfo splitb = 417 createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), HConstants.EMPTY_BYTE_ARRAY); 418 Thread.sleep(1001); 419 420 // Parent has daughters splita and splitb. Splita has daughters splitaa and splitab. 421 final Map<RegionInfo, Result> splitParents = new TreeMap<>(new SplitParentFirstComparator()); 422 splitParents.put(parent, createResult(parent, splita, splitb)); 423 // simulate that splita goes offline when it is split 424 splita = RegionInfoBuilder.newBuilder(splita).setOffline(true).build(); 425 splitParents.put(splita, createResult(splita, splitaa, splitab)); 426 427 final Map<RegionInfo, Result> mergedRegions = new TreeMap<>(); 428 CatalogJanitor spy = spy(this.janitor); 429 430 CatalogJanitorReport report = new CatalogJanitorReport(); 431 report.count = 10; 432 report.mergedRegions.putAll(mergedRegions); 433 report.splitParents.putAll(splitParents); 434 435 doReturn(report).when(spy).scanForReport(); 436 437 // Create ref from splita to parent 438 LOG.info("parent=" + parent.getShortNameToLog() + ", splita=" + splita.getShortNameToLog()); 439 Path splitaRef = 440 createReferences(this.masterServices, td, parent, splita, Bytes.toBytes("ccc"), false); 441 LOG.info("Created reference " + splitaRef); 442 443 // Parent and splita should not be removed because a reference from splita to parent. 444 int gcs = spy.scan(); 445 assertEquals(0, gcs); 446 447 // Now delete the ref 448 FileSystem fs = FileSystem.get(HTU.getConfiguration()); 449 assertTrue(fs.delete(splitaRef, true)); 450 451 // now, both parent, and splita can be deleted 452 gcs = spy.scan(); 453 assertEquals(2, gcs); 454 } 455 456 /** 457 * Test that we correctly archive all the storefiles when a region is deleted 458 */ 459 @Test 460 public void testSplitParentFirstComparator() { 461 SplitParentFirstComparator comp = new SplitParentFirstComparator(); 462 TableDescriptor td = createTableDescriptorForCurrentMethod(); 463 464 /* 465 * Region splits: rootRegion --- firstRegion --- firstRegiona | |- firstRegionb | |- lastRegion 466 * --- lastRegiona --- lastRegionaa | |- lastRegionab |- lastRegionb rootRegion : [] - [] 467 * firstRegion : [] - bbb lastRegion : bbb - [] firstRegiona : [] - aaa firstRegionb : aaa - bbb 468 * lastRegiona : bbb - ddd lastRegionb : ddd - [] 469 */ 470 471 // root region 472 RegionInfo rootRegion = createRegionInfo(td.getTableName(), HConstants.EMPTY_START_ROW, 473 HConstants.EMPTY_END_ROW, true); 474 RegionInfo firstRegion = 475 createRegionInfo(td.getTableName(), HConstants.EMPTY_START_ROW, Bytes.toBytes("bbb"), true); 476 RegionInfo lastRegion = 477 createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), HConstants.EMPTY_END_ROW, true); 478 479 assertTrue(comp.compare(rootRegion, rootRegion) == 0); 480 assertTrue(comp.compare(firstRegion, firstRegion) == 0); 481 assertTrue(comp.compare(lastRegion, lastRegion) == 0); 482 assertTrue(comp.compare(rootRegion, firstRegion) < 0); 483 assertTrue(comp.compare(rootRegion, lastRegion) < 0); 484 assertTrue(comp.compare(firstRegion, lastRegion) < 0); 485 486 // first region split into a, b 487 RegionInfo firstRegiona = 488 createRegionInfo(td.getTableName(), HConstants.EMPTY_START_ROW, Bytes.toBytes("aaa"), true); 489 RegionInfo firstRegionb = 490 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), true); 491 // last region split into a, b 492 RegionInfo lastRegiona = 493 createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ddd"), true); 494 RegionInfo lastRegionb = 495 createRegionInfo(td.getTableName(), Bytes.toBytes("ddd"), HConstants.EMPTY_END_ROW, true); 496 497 assertTrue(comp.compare(firstRegiona, firstRegiona) == 0); 498 assertTrue(comp.compare(firstRegionb, firstRegionb) == 0); 499 assertTrue(comp.compare(rootRegion, firstRegiona) < 0); 500 assertTrue(comp.compare(rootRegion, firstRegionb) < 0); 501 assertTrue(comp.compare(firstRegion, firstRegiona) < 0); 502 assertTrue(comp.compare(firstRegion, firstRegionb) < 0); 503 assertTrue(comp.compare(firstRegiona, firstRegionb) < 0); 504 505 assertTrue(comp.compare(lastRegiona, lastRegiona) == 0); 506 assertTrue(comp.compare(lastRegionb, lastRegionb) == 0); 507 assertTrue(comp.compare(rootRegion, lastRegiona) < 0); 508 assertTrue(comp.compare(rootRegion, lastRegionb) < 0); 509 assertTrue(comp.compare(lastRegion, lastRegiona) < 0); 510 assertTrue(comp.compare(lastRegion, lastRegionb) < 0); 511 assertTrue(comp.compare(lastRegiona, lastRegionb) < 0); 512 513 assertTrue(comp.compare(firstRegiona, lastRegiona) < 0); 514 assertTrue(comp.compare(firstRegiona, lastRegionb) < 0); 515 assertTrue(comp.compare(firstRegionb, lastRegiona) < 0); 516 assertTrue(comp.compare(firstRegionb, lastRegionb) < 0); 517 518 RegionInfo lastRegionaa = 519 createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false); 520 RegionInfo lastRegionab = 521 createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), false); 522 523 assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0); 524 assertTrue(comp.compare(lastRegiona, lastRegionab) < 0); 525 assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0); 526 } 527 528 @Test 529 public void testArchiveOldRegion() throws Exception { 530 // Create regions. 531 TableDescriptor td = createTableDescriptorForCurrentMethod(); 532 RegionInfo parent = 533 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee")); 534 RegionInfo splita = 535 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc")); 536 RegionInfo splitb = 537 createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee")); 538 539 // Test that when both daughter regions are in place, that we do not 540 // remove the parent. 541 Result parentMetaRow = createResult(parent, splita, splitb); 542 FileSystem fs = FileSystem.get(HTU.getConfiguration()); 543 Path rootdir = this.masterServices.getMasterFileSystem().getRootDir(); 544 // have to set the root directory since we use it in HFileDisposer to figure out to get to the 545 // archive directory. Otherwise, it just seems to pick the first root directory it can find (so 546 // the single test passes, but when the full suite is run, things get borked). 547 CommonFSUtils.setRootDir(fs.getConf(), rootdir); 548 Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName()); 549 Path storedir = 550 HRegionFileSystem.getStoreHomedir(tabledir, parent, td.getColumnFamilies()[0].getName()); 551 Path storeArchive = HFileArchiveUtil.getStoreArchivePath(this.masterServices.getConfiguration(), 552 parent, tabledir, td.getColumnFamilies()[0].getName()); 553 LOG.debug("Table dir:" + tabledir); 554 LOG.debug("Store dir:" + storedir); 555 LOG.debug("Store archive dir:" + storeArchive); 556 557 // add a couple of store files that we can check for 558 FileStatus[] mockFiles = addMockStoreFiles(2, this.masterServices, storedir); 559 // get the current store files for comparison 560 FileStatus[] storeFiles = fs.listStatus(storedir); 561 int index = 0; 562 for (FileStatus file : storeFiles) { 563 LOG.debug("Have store file:" + file.getPath()); 564 assertEquals("Got unexpected store file", mockFiles[index].getPath(), 565 storeFiles[index].getPath()); 566 index++; 567 } 568 569 // do the cleaning of the parent 570 assertTrue(CatalogJanitor.cleanParent(masterServices, parent, parentMetaRow)); 571 Path parentDir = new Path(tabledir, parent.getEncodedName()); 572 // Cleanup procedure runs async. Wait till it done. 573 ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor()); 574 assertTrue(!fs.exists(parentDir)); 575 LOG.debug("Finished cleanup of parent region"); 576 577 // and now check to make sure that the files have actually been archived 578 FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive); 579 logFiles("archived files", storeFiles); 580 logFiles("archived files", archivedStoreFiles); 581 582 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs); 583 584 // cleanup 585 CommonFSUtils.delete(fs, rootdir, true); 586 } 587 588 /** 589 * @param description description of the files for logging 590 * @param storeFiles the status of the files to log 591 */ 592 private void logFiles(String description, FileStatus[] storeFiles) { 593 LOG.debug("Current " + description + ": "); 594 for (FileStatus file : storeFiles) { 595 LOG.debug(Objects.toString(file.getPath())); 596 } 597 } 598 599 /** 600 * Test that if a store file with the same name is present as those already backed up cause the 601 * already archived files to be timestamped backup 602 */ 603 @Test 604 public void testDuplicateHFileResolution() throws Exception { 605 TableDescriptor td = createTableDescriptorForCurrentMethod(); 606 607 // Create regions. 608 RegionInfo parent = 609 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee")); 610 RegionInfo splita = 611 createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc")); 612 RegionInfo splitb = 613 createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee")); 614 // Test that when both daughter regions are in place, that we do not 615 // remove the parent. 616 Result r = createResult(parent, splita, splitb); 617 FileSystem fs = FileSystem.get(HTU.getConfiguration()); 618 Path rootdir = this.masterServices.getMasterFileSystem().getRootDir(); 619 // Have to set the root directory since we use it in HFileDisposer to figure out to get to the 620 // archive directory. Otherwise, it just seems to pick the first root directory it can find (so 621 // the single test passes, but when the full suite is run, things get borked). 622 CommonFSUtils.setRootDir(fs.getConf(), rootdir); 623 Path tabledir = CommonFSUtils.getTableDir(rootdir, parent.getTable()); 624 Path storedir = 625 HRegionFileSystem.getStoreHomedir(tabledir, parent, td.getColumnFamilies()[0].getName()); 626 LOG.info("Old root:" + rootdir); 627 LOG.info("Old table:" + tabledir); 628 LOG.info("Old store:" + storedir); 629 630 Path storeArchive = HFileArchiveUtil.getStoreArchivePath(this.masterServices.getConfiguration(), 631 parent, tabledir, td.getColumnFamilies()[0].getName()); 632 LOG.info("Old archive:" + storeArchive); 633 634 // enable archiving, make sure that files get archived 635 addMockStoreFiles(2, this.masterServices, storedir); 636 // get the current store files for comparison 637 FileStatus[] storeFiles = fs.listStatus(storedir); 638 // Do the cleaning of the parent 639 assertTrue(CatalogJanitor.cleanParent(masterServices, parent, r)); 640 Path parentDir = new Path(tabledir, parent.getEncodedName()); 641 ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor()); 642 assertTrue(!fs.exists(parentDir)); 643 644 // And now check to make sure that the files have actually been archived 645 FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive); 646 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs); 647 648 // now add store files with the same names as before to check backup 649 // enable archiving, make sure that files get archived 650 addMockStoreFiles(2, this.masterServices, storedir); 651 652 // Do the cleaning of the parent 653 assertTrue(CatalogJanitor.cleanParent(masterServices, parent, r)); 654 // Cleanup procedure runs async. Wait till it done. 655 ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor()); 656 assertTrue(!fs.exists(parentDir)); 657 658 // and now check to make sure that the files have actually been archived 659 archivedStoreFiles = fs.listStatus(storeArchive); 660 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true); 661 } 662 663 @Test 664 public void testAlreadyRunningStatus() throws Exception { 665 int numberOfThreads = 2; 666 List<Integer> gcValues = new ArrayList<>(); 667 Thread[] threads = new Thread[numberOfThreads]; 668 for (int i = 0; i < numberOfThreads; i++) { 669 threads[i] = new Thread(() -> { 670 try { 671 gcValues.add(janitor.scan()); 672 } catch (IOException e) { 673 throw new RuntimeException(e); 674 } 675 }); 676 } 677 for (int i = 0; i < numberOfThreads; i++) { 678 threads[i].start(); 679 } 680 for (int i = 0; i < numberOfThreads; i++) { 681 threads[i].join(); 682 } 683 assertTrue("One janitor.scan() call should have returned -1", gcValues.contains(-1)); 684 } 685 686 private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir) 687 throws IOException { 688 // get the existing store files 689 FileSystem fs = services.getMasterFileSystem().getFileSystem(); 690 fs.mkdirs(storedir); 691 // create the store files in the parent 692 for (int i = 0; i < count; i++) { 693 Path storeFile = new Path(storedir, "_store" + i); 694 FSDataOutputStream dos = fs.create(storeFile, true); 695 dos.writeBytes("Some data: " + i); 696 dos.close(); 697 } 698 LOG.debug("Adding " + count + " store files to the storedir:" + storedir); 699 // make sure the mock store files are there 700 FileStatus[] storeFiles = fs.listStatus(storedir); 701 assertEquals("Didn't have expected store files", count, storeFiles.length); 702 return storeFiles; 703 } 704 705 private String setRootDirAndCleanIt(final HBaseTestingUtil htu, final String subdir) 706 throws IOException { 707 Path testdir = htu.getDataTestDir(subdir); 708 FileSystem fs = FileSystem.get(htu.getConfiguration()); 709 if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true)); 710 CommonFSUtils.setRootDir(htu.getConfiguration(), testdir); 711 return CommonFSUtils.getRootDir(htu.getConfiguration()).toString(); 712 } 713 714 private Path createReferences(final MasterServices services, final TableDescriptor td, 715 final RegionInfo parent, final RegionInfo daughter, final byte[] midkey, final boolean top) 716 throws IOException { 717 Path rootdir = services.getMasterFileSystem().getRootDir(); 718 Path tabledir = CommonFSUtils.getTableDir(rootdir, parent.getTable()); 719 Path storedir = 720 HRegionFileSystem.getStoreHomedir(tabledir, daughter, td.getColumnFamilies()[0].getName()); 721 Reference ref = 722 top ? Reference.createTopReference(midkey) : Reference.createBottomReference(midkey); 723 long now = EnvironmentEdgeManager.currentTime(); 724 // Reference name has this format: StoreFile#REF_NAME_PARSER 725 Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName()); 726 FileSystem fs = services.getMasterFileSystem().getFileSystem(); 727 ref.write(fs, p); 728 return p; 729 } 730 731 private Result createResult(final RegionInfo parent, final RegionInfo a, final RegionInfo b) 732 throws IOException { 733 return MetaMockingUtil.getMetaTableRowResult(parent, null, a, b); 734 } 735}