001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.janitor;
019
020import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024import static org.mockito.Mockito.doReturn;
025import static org.mockito.Mockito.spy;
026
027import java.io.IOException;
028import java.util.ArrayList;
029import java.util.List;
030import java.util.Map;
031import java.util.NavigableMap;
032import java.util.Objects;
033import java.util.SortedMap;
034import java.util.SortedSet;
035import java.util.TreeMap;
036import java.util.concurrent.ConcurrentSkipListMap;
037import org.apache.hadoop.fs.FSDataOutputStream;
038import org.apache.hadoop.fs.FileStatus;
039import org.apache.hadoop.fs.FileSystem;
040import org.apache.hadoop.fs.Path;
041import org.apache.hadoop.hbase.HBaseClassTestRule;
042import org.apache.hadoop.hbase.HBaseTestingUtil;
043import org.apache.hadoop.hbase.HConstants;
044import org.apache.hadoop.hbase.MetaMockingUtil;
045import org.apache.hadoop.hbase.ServerName;
046import org.apache.hadoop.hbase.TableName;
047import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
048import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
049import org.apache.hadoop.hbase.client.RegionInfo;
050import org.apache.hadoop.hbase.client.RegionInfoBuilder;
051import org.apache.hadoop.hbase.client.Result;
052import org.apache.hadoop.hbase.client.TableDescriptor;
053import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
054import org.apache.hadoop.hbase.io.Reference;
055import org.apache.hadoop.hbase.master.MasterServices;
056import org.apache.hadoop.hbase.master.assignment.MockMasterServices;
057import org.apache.hadoop.hbase.master.janitor.CatalogJanitor.SplitParentFirstComparator;
058import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
059import org.apache.hadoop.hbase.regionserver.ChunkCreator;
060import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
061import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
062import org.apache.hadoop.hbase.testclassification.MasterTests;
063import org.apache.hadoop.hbase.testclassification.MediumTests;
064import org.apache.hadoop.hbase.util.Bytes;
065import org.apache.hadoop.hbase.util.CommonFSUtils;
066import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
067import org.apache.hadoop.hbase.util.HFileArchiveUtil;
068import org.apache.zookeeper.KeeperException;
069import org.junit.After;
070import org.junit.Before;
071import org.junit.BeforeClass;
072import org.junit.ClassRule;
073import org.junit.Rule;
074import org.junit.Test;
075import org.junit.experimental.categories.Category;
076import org.junit.rules.TestName;
077import org.slf4j.Logger;
078import org.slf4j.LoggerFactory;
079
080@Category({ MasterTests.class, MediumTests.class })
081public class TestCatalogJanitor {
082
083  @ClassRule
084  public static final HBaseClassTestRule CLASS_RULE =
085    HBaseClassTestRule.forClass(TestCatalogJanitor.class);
086
087  private static final Logger LOG = LoggerFactory.getLogger(TestCatalogJanitor.class);
088
089  private static final HBaseTestingUtil HTU = new HBaseTestingUtil();
090
091  @Rule
092  public final TestName name = new TestName();
093
094  private MockMasterServices masterServices;
095  private CatalogJanitor janitor;
096
097  @BeforeClass
098  public static void beforeClass() throws Exception {
099    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
100      MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
101  }
102
103  @Before
104  public void setup() throws IOException, KeeperException {
105    setRootDirAndCleanIt(HTU, this.name.getMethodName());
106    NavigableMap<ServerName, SortedSet<byte[]>> regionsToRegionServers =
107      new ConcurrentSkipListMap<ServerName, SortedSet<byte[]>>();
108    this.masterServices = new MockMasterServices(HTU.getConfiguration(), regionsToRegionServers);
109    this.masterServices.start(10, null);
110    this.janitor = new CatalogJanitor(masterServices);
111  }
112
113  @After
114  public void teardown() {
115    this.janitor.shutdown(true);
116    this.masterServices.stop("DONE");
117  }
118
119  private RegionInfo createRegionInfo(TableName tableName, byte[] startKey, byte[] endKey) {
120    return createRegionInfo(tableName, startKey, endKey, false);
121  }
122
123  private RegionInfo createRegionInfo(TableName tableName, byte[] startKey, byte[] endKey,
124    boolean split) {
125    return RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(endKey)
126      .setSplit(split).build();
127  }
128
129  /**
130   * Test clearing a split parent.
131   */
132  @Test
133  public void testCleanParent() throws IOException, InterruptedException {
134    TableDescriptor td = createTableDescriptorForCurrentMethod();
135    // Create regions.
136    RegionInfo parent =
137      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
138    RegionInfo splita =
139      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
140    RegionInfo splitb =
141      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
142    // Test that when both daughter regions are in place, that we do not remove the parent.
143    Result r = createResult(parent, splita, splitb);
144    // Add a reference under splitA directory so we don't clear out the parent.
145    Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
146    Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName());
147    Path parentdir = new Path(tabledir, parent.getEncodedName());
148    Path storedir = HRegionFileSystem.getStoreHomedir(tabledir, splita,
149      td.getColumnFamilies()[0].getName());
150    Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
151    long now = EnvironmentEdgeManager.currentTime();
152    // Reference name has this format: StoreFile#REF_NAME_PARSER
153    Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
154    FileSystem fs = this.masterServices.getMasterFileSystem().getFileSystem();
155    Path path = ref.write(fs, p);
156    assertTrue(fs.exists(path));
157    LOG.info("Created reference " + path);
158    // Add a parentdir for kicks so can check it gets removed by the catalogjanitor.
159    fs.mkdirs(parentdir);
160    assertFalse(CatalogJanitor.cleanParent(masterServices, parent, r));
161    ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
162    assertTrue(fs.exists(parentdir));
163    // Remove the reference file and try again.
164    assertTrue(fs.delete(p, true));
165    assertTrue(CatalogJanitor.cleanParent(masterServices, parent, r));
166    // Parent cleanup is run async as a procedure. Make sure parentdir is removed.
167    ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
168    assertTrue(!fs.exists(parentdir));
169  }
170
171  /**
172   * Make sure parent gets cleaned up even if daughter is cleaned up before it.
173   */
174  @Test
175  public void testParentCleanedEvenIfDaughterGoneFirst() throws IOException, InterruptedException {
176    parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(this.name.getMethodName(),
177      Bytes.toBytes("eee"));
178  }
179
180  /**
181   * Make sure last parent with empty end key gets cleaned up even if daughter is cleaned up before
182   * it.
183   */
184  @Test
185  public void testLastParentCleanedEvenIfDaughterGoneFirst()
186    throws IOException, InterruptedException {
187    parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(this.name.getMethodName(), new byte[0]);
188  }
189
190  /**
191   * @return A TableDescriptor with a tableName of current method name and a column family that is
192   *         MockMasterServices.DEFAULT_COLUMN_FAMILY_NAME)
193   */
194  private TableDescriptor createTableDescriptorForCurrentMethod() {
195    ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
196      .newBuilder(Bytes.toBytes(MockMasterServices.DEFAULT_COLUMN_FAMILY_NAME)).build();
197    return TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name.getMethodName()))
198      .setColumnFamily(columnFamilyDescriptor).build();
199  }
200
201  /**
202   * Make sure parent with specified end key gets cleaned up even if daughter is cleaned up before
203   * it.
204   * @param rootDir the test case name, used as the HBase testing utility root
205   * @param lastEndKey the end key of the split parent
206   */
207  private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(final String rootDir,
208    final byte[] lastEndKey) throws IOException, InterruptedException {
209    TableDescriptor td = createTableDescriptorForCurrentMethod();
210    // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
211    RegionInfo parent = createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), lastEndKey);
212    // Sleep a second else the encoded name on these regions comes out
213    // same for all with same start key and made in same second.
214    Thread.sleep(1001);
215
216    // Daughter a
217    RegionInfo splita =
218      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
219    Thread.sleep(1001);
220    // Make daughters of daughter a; splitaa and splitab.
221    RegionInfo splitaa =
222      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"));
223    RegionInfo splitab =
224      createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"));
225
226    // Daughter b
227    RegionInfo splitb = createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), lastEndKey);
228    Thread.sleep(1001);
229    // Make Daughters of daughterb; splitba and splitbb.
230    RegionInfo splitba =
231      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("ddd"));
232    RegionInfo splitbb = createRegionInfo(td.getTableName(), Bytes.toBytes("ddd"), lastEndKey);
233
234    // First test that our Comparator works right up in CatalogJanitor.
235    SortedMap<RegionInfo, Result> regions =
236      new TreeMap<>(new CatalogJanitor.SplitParentFirstComparator());
237    // Now make sure that this regions map sorts as we expect it to.
238    regions.put(parent, createResult(parent, splita, splitb));
239    regions.put(splitb, createResult(splitb, splitba, splitbb));
240    regions.put(splita, createResult(splita, splitaa, splitab));
241    // Assert its properly sorted.
242    int index = 0;
243    for (Map.Entry<RegionInfo, Result> e : regions.entrySet()) {
244      if (index == 0) {
245        assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName()));
246      } else if (index == 1) {
247        assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName()));
248      } else if (index == 2) {
249        assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName()));
250      }
251      index++;
252    }
253
254    // Now play around with the cleanParent function. Create a ref from splita up to the parent.
255    Path splitaRef =
256      createReferences(this.masterServices, td, parent, splita, Bytes.toBytes("ccc"), false);
257    // Make sure actual super parent sticks around because splita has a ref.
258    assertFalse(CatalogJanitor.cleanParent(masterServices, parent, regions.get(parent)));
259
260    // splitba, and split bb, do not have dirs in fs. That means that if
261    // we test splitb, it should get cleaned up.
262    assertTrue(CatalogJanitor.cleanParent(masterServices, splitb, regions.get(splitb)));
263
264    // Now remove ref from splita to parent... so parent can be let go and so
265    // the daughter splita can be split (can't split if still references).
266    // BUT make the timing such that the daughter gets cleaned up before we
267    // can get a chance to let go of the parent.
268    FileSystem fs = FileSystem.get(HTU.getConfiguration());
269    assertTrue(fs.delete(splitaRef, true));
270    // Create the refs from daughters of splita.
271    Path splitaaRef =
272      createReferences(this.masterServices, td, splita, splitaa, Bytes.toBytes("bbb"), false);
273    Path splitabRef =
274      createReferences(this.masterServices, td, splita, splitab, Bytes.toBytes("bbb"), true);
275
276    // Test splita. It should stick around because references from splitab, etc.
277    assertFalse(CatalogJanitor.cleanParent(masterServices, splita, regions.get(splita)));
278
279    // Now clean up parent daughter first. Remove references from its daughters.
280    assertTrue(fs.delete(splitaaRef, true));
281    assertTrue(fs.delete(splitabRef, true));
282    assertTrue(CatalogJanitor.cleanParent(masterServices, splita, regions.get(splita)));
283
284    // Super parent should get cleaned up now both splita and splitb are gone.
285    assertTrue(CatalogJanitor.cleanParent(masterServices, parent, regions.get(parent)));
286  }
287
288  /**
289   * CatalogJanitor.scan() should not clean parent regions if their own parents are still
290   * referencing them. This ensures that grandparent regions do not point to deleted parent regions.
291   */
292  @Test
293  public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
294    TableDescriptor td = createTableDescriptorForCurrentMethod();
295    // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
296
297    // Parent
298    RegionInfo parent =
299      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), HConstants.EMPTY_BYTE_ARRAY, true);
300    // Sleep a second else the encoded name on these regions comes out
301    // same for all with same start key and made in same second.
302    Thread.sleep(1001);
303
304    // Daughter a
305    RegionInfo splita =
306      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"), true);
307    Thread.sleep(1001);
308
309    // Make daughters of daughter a; splitaa and splitab.
310    RegionInfo splitaa =
311      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), false);
312    RegionInfo splitab =
313      createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false);
314
315    // Daughter b
316    RegionInfo splitb =
317      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), HConstants.EMPTY_BYTE_ARRAY);
318    Thread.sleep(1001);
319
320    // Parent has daughters splita and splitb. Splita has daughters splitaa and splitab.
321    final Map<RegionInfo, Result> splitParents = new TreeMap<>(new SplitParentFirstComparator());
322    splitParents.put(parent, createResult(parent, splita, splitb));
323    // simulate that splita goes offline when it is split
324    splita = RegionInfoBuilder.newBuilder(splita).setOffline(true).build();
325    splitParents.put(splita, createResult(splita, splitaa, splitab));
326
327    final Map<RegionInfo, Result> mergedRegions = new TreeMap<>();
328    CatalogJanitor spy = spy(this.janitor);
329
330    Report report = new Report();
331    report.count = 10;
332    report.mergedRegions.putAll(mergedRegions);
333    report.splitParents.putAll(splitParents);
334
335    doReturn(report).when(spy).scanForReport();
336
337    // Create ref from splita to parent
338    LOG.info("parent=" + parent.getShortNameToLog() + ", splita=" + splita.getShortNameToLog());
339    Path splitaRef =
340      createReferences(this.masterServices, td, parent, splita, Bytes.toBytes("ccc"), false);
341    LOG.info("Created reference " + splitaRef);
342
343    // Parent and splita should not be removed because a reference from splita to parent.
344    int gcs = spy.scan();
345    assertEquals(0, gcs);
346
347    // Now delete the ref
348    FileSystem fs = FileSystem.get(HTU.getConfiguration());
349    assertTrue(fs.delete(splitaRef, true));
350
351    // now, both parent, and splita can be deleted
352    gcs = spy.scan();
353    assertEquals(2, gcs);
354  }
355
356  /**
357   * Test that we correctly archive all the storefiles when a region is deleted
358   * @throws Exception
359   */
360  @Test
361  public void testSplitParentFirstComparator() {
362    SplitParentFirstComparator comp = new SplitParentFirstComparator();
363    TableDescriptor td = createTableDescriptorForCurrentMethod();
364
365    /*
366     * Region splits: rootRegion --- firstRegion --- firstRegiona | |- firstRegionb | |- lastRegion
367     * --- lastRegiona --- lastRegionaa | |- lastRegionab |- lastRegionb rootRegion : [] - []
368     * firstRegion : [] - bbb lastRegion : bbb - [] firstRegiona : [] - aaa firstRegionb : aaa - bbb
369     * lastRegiona : bbb - ddd lastRegionb : ddd - []
370     */
371
372    // root region
373    RegionInfo rootRegion = createRegionInfo(td.getTableName(), HConstants.EMPTY_START_ROW,
374      HConstants.EMPTY_END_ROW, true);
375    RegionInfo firstRegion =
376      createRegionInfo(td.getTableName(), HConstants.EMPTY_START_ROW, Bytes.toBytes("bbb"), true);
377    RegionInfo lastRegion =
378      createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), HConstants.EMPTY_END_ROW, true);
379
380    assertTrue(comp.compare(rootRegion, rootRegion) == 0);
381    assertTrue(comp.compare(firstRegion, firstRegion) == 0);
382    assertTrue(comp.compare(lastRegion, lastRegion) == 0);
383    assertTrue(comp.compare(rootRegion, firstRegion) < 0);
384    assertTrue(comp.compare(rootRegion, lastRegion) < 0);
385    assertTrue(comp.compare(firstRegion, lastRegion) < 0);
386
387    // first region split into a, b
388    RegionInfo firstRegiona =
389      createRegionInfo(td.getTableName(), HConstants.EMPTY_START_ROW, Bytes.toBytes("aaa"), true);
390    RegionInfo firstRegionb =
391      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), true);
392    // last region split into a, b
393    RegionInfo lastRegiona =
394      createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ddd"), true);
395    RegionInfo lastRegionb =
396      createRegionInfo(td.getTableName(), Bytes.toBytes("ddd"), HConstants.EMPTY_END_ROW, true);
397
398    assertTrue(comp.compare(firstRegiona, firstRegiona) == 0);
399    assertTrue(comp.compare(firstRegionb, firstRegionb) == 0);
400    assertTrue(comp.compare(rootRegion, firstRegiona) < 0);
401    assertTrue(comp.compare(rootRegion, firstRegionb) < 0);
402    assertTrue(comp.compare(firstRegion, firstRegiona) < 0);
403    assertTrue(comp.compare(firstRegion, firstRegionb) < 0);
404    assertTrue(comp.compare(firstRegiona, firstRegionb) < 0);
405
406    assertTrue(comp.compare(lastRegiona, lastRegiona) == 0);
407    assertTrue(comp.compare(lastRegionb, lastRegionb) == 0);
408    assertTrue(comp.compare(rootRegion, lastRegiona) < 0);
409    assertTrue(comp.compare(rootRegion, lastRegionb) < 0);
410    assertTrue(comp.compare(lastRegion, lastRegiona) < 0);
411    assertTrue(comp.compare(lastRegion, lastRegionb) < 0);
412    assertTrue(comp.compare(lastRegiona, lastRegionb) < 0);
413
414    assertTrue(comp.compare(firstRegiona, lastRegiona) < 0);
415    assertTrue(comp.compare(firstRegiona, lastRegionb) < 0);
416    assertTrue(comp.compare(firstRegionb, lastRegiona) < 0);
417    assertTrue(comp.compare(firstRegionb, lastRegionb) < 0);
418
419    RegionInfo lastRegionaa =
420      createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false);
421    RegionInfo lastRegionab =
422      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), false);
423
424    assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0);
425    assertTrue(comp.compare(lastRegiona, lastRegionab) < 0);
426    assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0);
427  }
428
429  @Test
430  public void testArchiveOldRegion() throws Exception {
431    // Create regions.
432    TableDescriptor td = createTableDescriptorForCurrentMethod();
433    RegionInfo parent =
434      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
435    RegionInfo splita =
436      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
437    RegionInfo splitb =
438      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
439
440    // Test that when both daughter regions are in place, that we do not
441    // remove the parent.
442    Result parentMetaRow = createResult(parent, splita, splitb);
443    FileSystem fs = FileSystem.get(HTU.getConfiguration());
444    Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
445    // have to set the root directory since we use it in HFileDisposer to figure out to get to the
446    // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
447    // the single test passes, but when the full suite is run, things get borked).
448    CommonFSUtils.setRootDir(fs.getConf(), rootdir);
449    Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName());
450    Path storedir = HRegionFileSystem.getStoreHomedir(tabledir, parent,
451      td.getColumnFamilies()[0].getName());
452    Path storeArchive = HFileArchiveUtil.getStoreArchivePath(this.masterServices.getConfiguration(),
453      parent, tabledir, td.getColumnFamilies()[0].getName());
454    LOG.debug("Table dir:" + tabledir);
455    LOG.debug("Store dir:" + storedir);
456    LOG.debug("Store archive dir:" + storeArchive);
457
458    // add a couple of store files that we can check for
459    FileStatus[] mockFiles = addMockStoreFiles(2, this.masterServices, storedir);
460    // get the current store files for comparison
461    FileStatus[] storeFiles = fs.listStatus(storedir);
462    int index = 0;
463    for (FileStatus file : storeFiles) {
464      LOG.debug("Have store file:" + file.getPath());
465      assertEquals("Got unexpected store file", mockFiles[index].getPath(),
466        storeFiles[index].getPath());
467      index++;
468    }
469
470    // do the cleaning of the parent
471    assertTrue(CatalogJanitor.cleanParent(masterServices, parent, parentMetaRow));
472    Path parentDir = new Path(tabledir, parent.getEncodedName());
473    // Cleanup procedure runs async. Wait till it done.
474    ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
475    assertTrue(!fs.exists(parentDir));
476    LOG.debug("Finished cleanup of parent region");
477
478    // and now check to make sure that the files have actually been archived
479    FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
480    logFiles("archived files", storeFiles);
481    logFiles("archived files", archivedStoreFiles);
482
483    assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
484
485    // cleanup
486    CommonFSUtils.delete(fs, rootdir, true);
487  }
488
489  /**
490   * @param description description of the files for logging
491   * @param storeFiles the status of the files to log
492   */
493  private void logFiles(String description, FileStatus[] storeFiles) {
494    LOG.debug("Current " + description + ": ");
495    for (FileStatus file : storeFiles) {
496      LOG.debug(Objects.toString(file.getPath()));
497    }
498  }
499
500  /**
501   * Test that if a store file with the same name is present as those already backed up cause the
502   * already archived files to be timestamped backup
503   */
504  @Test
505  public void testDuplicateHFileResolution() throws Exception {
506    TableDescriptor td = createTableDescriptorForCurrentMethod();
507
508    // Create regions.
509    RegionInfo parent =
510      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
511    RegionInfo splita =
512      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
513    RegionInfo splitb =
514      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
515    // Test that when both daughter regions are in place, that we do not
516    // remove the parent.
517    Result r = createResult(parent, splita, splitb);
518    FileSystem fs = FileSystem.get(HTU.getConfiguration());
519    Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
520    // Have to set the root directory since we use it in HFileDisposer to figure out to get to the
521    // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
522    // the single test passes, but when the full suite is run, things get borked).
523    CommonFSUtils.setRootDir(fs.getConf(), rootdir);
524    Path tabledir = CommonFSUtils.getTableDir(rootdir, parent.getTable());
525    Path storedir = HRegionFileSystem.getStoreHomedir(tabledir, parent,
526      td.getColumnFamilies()[0].getName());
527    LOG.info("Old root:" + rootdir);
528    LOG.info("Old table:" + tabledir);
529    LOG.info("Old store:" + storedir);
530
531    Path storeArchive = HFileArchiveUtil.getStoreArchivePath(this.masterServices.getConfiguration(),
532      parent, tabledir, td.getColumnFamilies()[0].getName());
533    LOG.info("Old archive:" + storeArchive);
534
535    // enable archiving, make sure that files get archived
536    addMockStoreFiles(2, this.masterServices, storedir);
537    // get the current store files for comparison
538    FileStatus[] storeFiles = fs.listStatus(storedir);
539    // Do the cleaning of the parent
540    assertTrue(CatalogJanitor.cleanParent(masterServices, parent, r));
541    Path parentDir = new Path(tabledir, parent.getEncodedName());
542    ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
543    assertTrue(!fs.exists(parentDir));
544
545    // And now check to make sure that the files have actually been archived
546    FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
547    assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
548
549    // now add store files with the same names as before to check backup
550    // enable archiving, make sure that files get archived
551    addMockStoreFiles(2, this.masterServices, storedir);
552
553    // Do the cleaning of the parent
554    assertTrue(CatalogJanitor.cleanParent(masterServices, parent, r));
555    // Cleanup procedure runs async. Wait till it done.
556    ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
557    assertTrue(!fs.exists(parentDir));
558
559    // and now check to make sure that the files have actually been archived
560    archivedStoreFiles = fs.listStatus(storeArchive);
561    assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true);
562  }
563
564  @Test
565  public void testAlreadyRunningStatus() throws Exception {
566    int numberOfThreads = 2;
567    List<Integer> gcValues = new ArrayList<>();
568    Thread[] threads = new Thread[numberOfThreads];
569    for (int i = 0; i < numberOfThreads; i++) {
570      threads[i] = new Thread(() -> {
571        try {
572          gcValues.add(janitor.scan());
573        } catch (IOException e) {
574          throw new RuntimeException(e);
575        }
576      });
577    }
578    for (int i = 0; i < numberOfThreads; i++) {
579      threads[i].start();
580    }
581    for (int i = 0; i < numberOfThreads; i++) {
582      threads[i].join();
583    }
584    assertTrue("One janitor.scan() call should have returned -1", gcValues.contains(-1));
585  }
586
587  private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
588    throws IOException {
589    // get the existing store files
590    FileSystem fs = services.getMasterFileSystem().getFileSystem();
591    fs.mkdirs(storedir);
592    // create the store files in the parent
593    for (int i = 0; i < count; i++) {
594      Path storeFile = new Path(storedir, "_store" + i);
595      FSDataOutputStream dos = fs.create(storeFile, true);
596      dos.writeBytes("Some data: " + i);
597      dos.close();
598    }
599    LOG.debug("Adding " + count + " store files to the storedir:" + storedir);
600    // make sure the mock store files are there
601    FileStatus[] storeFiles = fs.listStatus(storedir);
602    assertEquals("Didn't have expected store files", count, storeFiles.length);
603    return storeFiles;
604  }
605
606  private String setRootDirAndCleanIt(final HBaseTestingUtil htu, final String subdir)
607    throws IOException {
608    Path testdir = htu.getDataTestDir(subdir);
609    FileSystem fs = FileSystem.get(htu.getConfiguration());
610    if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true));
611    CommonFSUtils.setRootDir(htu.getConfiguration(), testdir);
612    return CommonFSUtils.getRootDir(htu.getConfiguration()).toString();
613  }
614
615  private Path createReferences(final MasterServices services, final TableDescriptor td,
616    final RegionInfo parent, final RegionInfo daughter, final byte[] midkey, final boolean top)
617    throws IOException {
618    Path rootdir = services.getMasterFileSystem().getRootDir();
619    Path tabledir = CommonFSUtils.getTableDir(rootdir, parent.getTable());
620    Path storedir = HRegionFileSystem.getStoreHomedir(tabledir, daughter,
621      td.getColumnFamilies()[0].getName());
622    Reference ref =
623      top ? Reference.createTopReference(midkey) : Reference.createBottomReference(midkey);
624    long now = EnvironmentEdgeManager.currentTime();
625    // Reference name has this format: StoreFile#REF_NAME_PARSER
626    Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
627    FileSystem fs = services.getMasterFileSystem().getFileSystem();
628    ref.write(fs, p);
629    return p;
630  }
631
632  private Result createResult(final RegionInfo parent, final RegionInfo a, final RegionInfo b)
633    throws IOException {
634    return MetaMockingUtil.getMetaTableRowResult(parent, null, a, b);
635  }
636}