001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.janitor;
019
020import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024import static org.mockito.Mockito.any;
025import static org.mockito.Mockito.doAnswer;
026import static org.mockito.Mockito.doReturn;
027import static org.mockito.Mockito.doThrow;
028import static org.mockito.Mockito.spy;
029import static org.mockito.Mockito.when;
030
031import java.io.IOException;
032import java.util.ArrayList;
033import java.util.List;
034import java.util.Map;
035import java.util.NavigableMap;
036import java.util.Objects;
037import java.util.SortedMap;
038import java.util.SortedSet;
039import java.util.TreeMap;
040import java.util.concurrent.ConcurrentSkipListMap;
041import java.util.concurrent.atomic.AtomicBoolean;
042import org.apache.hadoop.fs.FSDataOutputStream;
043import org.apache.hadoop.fs.FileStatus;
044import org.apache.hadoop.fs.FileSystem;
045import org.apache.hadoop.fs.Path;
046import org.apache.hadoop.hbase.HBaseClassTestRule;
047import org.apache.hadoop.hbase.HBaseTestingUtil;
048import org.apache.hadoop.hbase.HConstants;
049import org.apache.hadoop.hbase.MetaMockingUtil;
050import org.apache.hadoop.hbase.ServerName;
051import org.apache.hadoop.hbase.TableName;
052import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
053import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
054import org.apache.hadoop.hbase.client.RegionInfo;
055import org.apache.hadoop.hbase.client.RegionInfoBuilder;
056import org.apache.hadoop.hbase.client.Result;
057import org.apache.hadoop.hbase.client.TableDescriptor;
058import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
059import org.apache.hadoop.hbase.io.Reference;
060import org.apache.hadoop.hbase.master.MasterFileSystem;
061import org.apache.hadoop.hbase.master.MasterServices;
062import org.apache.hadoop.hbase.master.assignment.MockMasterServices;
063import org.apache.hadoop.hbase.master.janitor.CatalogJanitor.SplitParentFirstComparator;
064import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
065import org.apache.hadoop.hbase.regionserver.ChunkCreator;
066import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
067import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
068import org.apache.hadoop.hbase.testclassification.MasterTests;
069import org.apache.hadoop.hbase.testclassification.MediumTests;
070import org.apache.hadoop.hbase.util.Bytes;
071import org.apache.hadoop.hbase.util.CommonFSUtils;
072import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
073import org.apache.hadoop.hbase.util.HFileArchiveUtil;
074import org.apache.zookeeper.KeeperException;
075import org.junit.After;
076import org.junit.Before;
077import org.junit.BeforeClass;
078import org.junit.ClassRule;
079import org.junit.Rule;
080import org.junit.Test;
081import org.junit.experimental.categories.Category;
082import org.junit.rules.TestName;
083import org.slf4j.Logger;
084import org.slf4j.LoggerFactory;
085
086@Category({ MasterTests.class, MediumTests.class })
087public class TestCatalogJanitor {
088
089  @ClassRule
090  public static final HBaseClassTestRule CLASS_RULE =
091    HBaseClassTestRule.forClass(TestCatalogJanitor.class);
092
093  private static final Logger LOG = LoggerFactory.getLogger(TestCatalogJanitor.class);
094
095  private static final HBaseTestingUtil HTU = new HBaseTestingUtil();
096
097  @Rule
098  public final TestName name = new TestName();
099
100  private MockMasterServices masterServices;
101  private CatalogJanitor janitor;
102
103  @BeforeClass
104  public static void beforeClass() throws Exception {
105    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
106      MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
107  }
108
109  @Before
110  public void setup() throws IOException, KeeperException {
111    setRootDirAndCleanIt(HTU, this.name.getMethodName());
112    NavigableMap<ServerName, SortedSet<byte[]>> regionsToRegionServers =
113      new ConcurrentSkipListMap<ServerName, SortedSet<byte[]>>();
114    this.masterServices = new MockMasterServices(HTU.getConfiguration(), regionsToRegionServers);
115    this.masterServices.start(10, null);
116    this.janitor = new CatalogJanitor(masterServices);
117  }
118
119  @After
120  public void teardown() {
121    this.janitor.shutdown(true);
122    this.masterServices.stop("DONE");
123  }
124
125  private RegionInfo createRegionInfo(TableName tableName, byte[] startKey, byte[] endKey) {
126    return createRegionInfo(tableName, startKey, endKey, false);
127  }
128
129  private RegionInfo createRegionInfo(TableName tableName, byte[] startKey, byte[] endKey,
130    boolean split) {
131    return RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(endKey)
132      .setSplit(split).build();
133  }
134
135  @Test
136  public void testCleanMerge() throws IOException {
137    TableDescriptor td = createTableDescriptorForCurrentMethod();
138    // Create regions.
139    RegionInfo merged =
140      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
141    RegionInfo parenta =
142      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
143    RegionInfo parentb =
144      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
145
146    List<RegionInfo> parents = new ArrayList<>();
147    parents.add(parenta);
148    parents.add(parentb);
149
150    Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
151    Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName());
152    Path storedir =
153      HRegionFileSystem.getStoreHomedir(tabledir, merged, td.getColumnFamilies()[0].getName());
154
155    Path parentaRef = createMergeReferenceFile(storedir, merged, parenta);
156    Path parentbRef = createMergeReferenceFile(storedir, merged, parentb);
157
158    // references exist, should not clean
159    assertFalse(CatalogJanitor.cleanMergeRegion(masterServices, merged, parents));
160
161    masterServices.getMasterFileSystem().getFileSystem().delete(parentaRef, false);
162
163    // one reference still exists, should not clean
164    assertFalse(CatalogJanitor.cleanMergeRegion(masterServices, merged, parents));
165
166    masterServices.getMasterFileSystem().getFileSystem().delete(parentbRef, false);
167
168    // all references removed, should clean
169    assertTrue(CatalogJanitor.cleanMergeRegion(masterServices, merged, parents));
170  }
171
172  @Test
173  public void testDontCleanMergeIfFileSystemException() throws IOException {
174    TableDescriptor td = createTableDescriptorForCurrentMethod();
175    // Create regions.
176    RegionInfo merged =
177      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
178    RegionInfo parenta =
179      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
180    RegionInfo parentb =
181      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
182
183    List<RegionInfo> parents = new ArrayList<>();
184    parents.add(parenta);
185    parents.add(parentb);
186
187    Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
188    Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName());
189    Path storedir =
190      HRegionFileSystem.getStoreHomedir(tabledir, merged, td.getColumnFamilies()[0].getName());
191    createMergeReferenceFile(storedir, merged, parenta);
192
193    MasterServices mockedMasterServices = spy(masterServices);
194    MasterFileSystem mockedMasterFileSystem = spy(masterServices.getMasterFileSystem());
195    FileSystem mockedFileSystem = spy(masterServices.getMasterFileSystem().getFileSystem());
196
197    when(mockedMasterServices.getMasterFileSystem()).thenReturn(mockedMasterFileSystem);
198    when(mockedMasterFileSystem.getFileSystem()).thenReturn(mockedFileSystem);
199
200    // throw on the first exists check
201    doThrow(new IOException("Some exception")).when(mockedFileSystem).exists(any());
202
203    assertFalse(CatalogJanitor.cleanMergeRegion(mockedMasterServices, merged, parents));
204
205    // throw on the second exists check (within HRegionfileSystem)
206    AtomicBoolean returned = new AtomicBoolean(false);
207    doAnswer(invocationOnMock -> {
208      if (!returned.get()) {
209        returned.set(true);
210        return masterServices.getMasterFileSystem().getFileSystem()
211          .exists(invocationOnMock.getArgument(0));
212      }
213      throw new IOException("Some exception");
214    }).when(mockedFileSystem).exists(any());
215
216    assertFalse(CatalogJanitor.cleanMergeRegion(mockedMasterServices, merged, parents));
217  }
218
219  private Path createMergeReferenceFile(Path storeDir, RegionInfo mergedRegion,
220    RegionInfo parentRegion) throws IOException {
221    Reference ref = Reference.createTopReference(mergedRegion.getStartKey());
222    long now = EnvironmentEdgeManager.currentTime();
223    // Reference name has this format: StoreFile#REF_NAME_PARSER
224    Path p = new Path(storeDir, Long.toString(now) + "." + parentRegion.getEncodedName());
225    FileSystem fs = this.masterServices.getMasterFileSystem().getFileSystem();
226    return ref.write(fs, p);
227  }
228
229  /**
230   * Test clearing a split parent.
231   */
232  @Test
233  public void testCleanParent() throws IOException, InterruptedException {
234    TableDescriptor td = createTableDescriptorForCurrentMethod();
235    // Create regions.
236    RegionInfo parent =
237      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
238    RegionInfo splita =
239      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
240    RegionInfo splitb =
241      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
242    // Test that when both daughter regions are in place, that we do not remove the parent.
243    Result r = createResult(parent, splita, splitb);
244    // Add a reference under splitA directory so we don't clear out the parent.
245    Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
246    Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName());
247    Path parentdir = new Path(tabledir, parent.getEncodedName());
248    Path storedir =
249      HRegionFileSystem.getStoreHomedir(tabledir, splita, td.getColumnFamilies()[0].getName());
250    Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
251    long now = EnvironmentEdgeManager.currentTime();
252    // Reference name has this format: StoreFile#REF_NAME_PARSER
253    Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
254    FileSystem fs = this.masterServices.getMasterFileSystem().getFileSystem();
255    Path path = ref.write(fs, p);
256    assertTrue(fs.exists(path));
257    LOG.info("Created reference " + path);
258    // Add a parentdir for kicks so can check it gets removed by the catalogjanitor.
259    fs.mkdirs(parentdir);
260    assertFalse(CatalogJanitor.cleanParent(masterServices, parent, r));
261    ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
262    assertTrue(fs.exists(parentdir));
263    // Remove the reference file and try again.
264    assertTrue(fs.delete(p, true));
265    assertTrue(CatalogJanitor.cleanParent(masterServices, parent, r));
266    // Parent cleanup is run async as a procedure. Make sure parentdir is removed.
267    ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
268    assertTrue(!fs.exists(parentdir));
269  }
270
271  /**
272   * Make sure parent gets cleaned up even if daughter is cleaned up before it.
273   */
274  @Test
275  public void testParentCleanedEvenIfDaughterGoneFirst() throws IOException, InterruptedException {
276    parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(this.name.getMethodName(),
277      Bytes.toBytes("eee"));
278  }
279
280  /**
281   * Make sure last parent with empty end key gets cleaned up even if daughter is cleaned up before
282   * it.
283   */
284  @Test
285  public void testLastParentCleanedEvenIfDaughterGoneFirst()
286    throws IOException, InterruptedException {
287    parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(this.name.getMethodName(), new byte[0]);
288  }
289
290  /**
291   * @return A TableDescriptor with a tableName of current method name and a column family that is
292   *         MockMasterServices.DEFAULT_COLUMN_FAMILY_NAME)
293   */
294  private TableDescriptor createTableDescriptorForCurrentMethod() {
295    ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
296      .newBuilder(Bytes.toBytes(MockMasterServices.DEFAULT_COLUMN_FAMILY_NAME)).build();
297    return TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name.getMethodName()))
298      .setColumnFamily(columnFamilyDescriptor).build();
299  }
300
301  /**
302   * Make sure parent with specified end key gets cleaned up even if daughter is cleaned up before
303   * it.
304   * @param rootDir    the test case name, used as the HBase testing utility root
305   * @param lastEndKey the end key of the split parent
306   */
307  private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(final String rootDir,
308    final byte[] lastEndKey) throws IOException, InterruptedException {
309    TableDescriptor td = createTableDescriptorForCurrentMethod();
310    // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
311    RegionInfo parent = createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), lastEndKey);
312    // Sleep a second else the encoded name on these regions comes out
313    // same for all with same start key and made in same second.
314    Thread.sleep(1001);
315
316    // Daughter a
317    RegionInfo splita =
318      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
319    Thread.sleep(1001);
320    // Make daughters of daughter a; splitaa and splitab.
321    RegionInfo splitaa =
322      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"));
323    RegionInfo splitab =
324      createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"));
325
326    // Daughter b
327    RegionInfo splitb = createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), lastEndKey);
328    Thread.sleep(1001);
329    // Make Daughters of daughterb; splitba and splitbb.
330    RegionInfo splitba =
331      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("ddd"));
332    RegionInfo splitbb = createRegionInfo(td.getTableName(), Bytes.toBytes("ddd"), lastEndKey);
333
334    // First test that our Comparator works right up in CatalogJanitor.
335    SortedMap<RegionInfo, Result> regions =
336      new TreeMap<>(new CatalogJanitor.SplitParentFirstComparator());
337    // Now make sure that this regions map sorts as we expect it to.
338    regions.put(parent, createResult(parent, splita, splitb));
339    regions.put(splitb, createResult(splitb, splitba, splitbb));
340    regions.put(splita, createResult(splita, splitaa, splitab));
341    // Assert its properly sorted.
342    int index = 0;
343    for (Map.Entry<RegionInfo, Result> e : regions.entrySet()) {
344      if (index == 0) {
345        assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName()));
346      } else if (index == 1) {
347        assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName()));
348      } else if (index == 2) {
349        assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName()));
350      }
351      index++;
352    }
353
354    // Now play around with the cleanParent function. Create a ref from splita up to the parent.
355    Path splitaRef =
356      createReferences(this.masterServices, td, parent, splita, Bytes.toBytes("ccc"), false);
357    // Make sure actual super parent sticks around because splita has a ref.
358    assertFalse(CatalogJanitor.cleanParent(masterServices, parent, regions.get(parent)));
359
360    // splitba, and split bb, do not have dirs in fs. That means that if
361    // we test splitb, it should get cleaned up.
362    assertTrue(CatalogJanitor.cleanParent(masterServices, splitb, regions.get(splitb)));
363
364    // Now remove ref from splita to parent... so parent can be let go and so
365    // the daughter splita can be split (can't split if still references).
366    // BUT make the timing such that the daughter gets cleaned up before we
367    // can get a chance to let go of the parent.
368    FileSystem fs = FileSystem.get(HTU.getConfiguration());
369    assertTrue(fs.delete(splitaRef, true));
370    // Create the refs from daughters of splita.
371    Path splitaaRef =
372      createReferences(this.masterServices, td, splita, splitaa, Bytes.toBytes("bbb"), false);
373    Path splitabRef =
374      createReferences(this.masterServices, td, splita, splitab, Bytes.toBytes("bbb"), true);
375
376    // Test splita. It should stick around because references from splitab, etc.
377    assertFalse(CatalogJanitor.cleanParent(masterServices, splita, regions.get(splita)));
378
379    // Now clean up parent daughter first. Remove references from its daughters.
380    assertTrue(fs.delete(splitaaRef, true));
381    assertTrue(fs.delete(splitabRef, true));
382    assertTrue(CatalogJanitor.cleanParent(masterServices, splita, regions.get(splita)));
383
384    // Super parent should get cleaned up now both splita and splitb are gone.
385    assertTrue(CatalogJanitor.cleanParent(masterServices, parent, regions.get(parent)));
386  }
387
388  /**
389   * CatalogJanitor.scan() should not clean parent regions if their own parents are still
390   * referencing them. This ensures that grandparent regions do not point to deleted parent regions.
391   */
392  @Test
393  public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
394    TableDescriptor td = createTableDescriptorForCurrentMethod();
395    // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
396
397    // Parent
398    RegionInfo parent =
399      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), HConstants.EMPTY_BYTE_ARRAY, true);
400    // Sleep a second else the encoded name on these regions comes out
401    // same for all with same start key and made in same second.
402    Thread.sleep(1001);
403
404    // Daughter a
405    RegionInfo splita =
406      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"), true);
407    Thread.sleep(1001);
408
409    // Make daughters of daughter a; splitaa and splitab.
410    RegionInfo splitaa =
411      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), false);
412    RegionInfo splitab =
413      createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false);
414
415    // Daughter b
416    RegionInfo splitb =
417      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), HConstants.EMPTY_BYTE_ARRAY);
418    Thread.sleep(1001);
419
420    // Parent has daughters splita and splitb. Splita has daughters splitaa and splitab.
421    final Map<RegionInfo, Result> splitParents = new TreeMap<>(new SplitParentFirstComparator());
422    splitParents.put(parent, createResult(parent, splita, splitb));
423    // simulate that splita goes offline when it is split
424    splita = RegionInfoBuilder.newBuilder(splita).setOffline(true).build();
425    splitParents.put(splita, createResult(splita, splitaa, splitab));
426
427    final Map<RegionInfo, Result> mergedRegions = new TreeMap<>();
428    CatalogJanitor spy = spy(this.janitor);
429
430    CatalogJanitorReport report = new CatalogJanitorReport();
431    report.count = 10;
432    report.mergedRegions.putAll(mergedRegions);
433    report.splitParents.putAll(splitParents);
434
435    doReturn(report).when(spy).scanForReport();
436
437    // Create ref from splita to parent
438    LOG.info("parent=" + parent.getShortNameToLog() + ", splita=" + splita.getShortNameToLog());
439    Path splitaRef =
440      createReferences(this.masterServices, td, parent, splita, Bytes.toBytes("ccc"), false);
441    LOG.info("Created reference " + splitaRef);
442
443    // Parent and splita should not be removed because a reference from splita to parent.
444    int gcs = spy.scan();
445    assertEquals(0, gcs);
446
447    // Now delete the ref
448    FileSystem fs = FileSystem.get(HTU.getConfiguration());
449    assertTrue(fs.delete(splitaRef, true));
450
451    // now, both parent, and splita can be deleted
452    gcs = spy.scan();
453    assertEquals(2, gcs);
454  }
455
456  /**
457   * Test that we correctly archive all the storefiles when a region is deleted
458   */
459  @Test
460  public void testSplitParentFirstComparator() {
461    SplitParentFirstComparator comp = new SplitParentFirstComparator();
462    TableDescriptor td = createTableDescriptorForCurrentMethod();
463
464    /*
465     * Region splits: rootRegion --- firstRegion --- firstRegiona | |- firstRegionb | |- lastRegion
466     * --- lastRegiona --- lastRegionaa | |- lastRegionab |- lastRegionb rootRegion : [] - []
467     * firstRegion : [] - bbb lastRegion : bbb - [] firstRegiona : [] - aaa firstRegionb : aaa - bbb
468     * lastRegiona : bbb - ddd lastRegionb : ddd - []
469     */
470
471    // root region
472    RegionInfo rootRegion = createRegionInfo(td.getTableName(), HConstants.EMPTY_START_ROW,
473      HConstants.EMPTY_END_ROW, true);
474    RegionInfo firstRegion =
475      createRegionInfo(td.getTableName(), HConstants.EMPTY_START_ROW, Bytes.toBytes("bbb"), true);
476    RegionInfo lastRegion =
477      createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), HConstants.EMPTY_END_ROW, true);
478
479    assertTrue(comp.compare(rootRegion, rootRegion) == 0);
480    assertTrue(comp.compare(firstRegion, firstRegion) == 0);
481    assertTrue(comp.compare(lastRegion, lastRegion) == 0);
482    assertTrue(comp.compare(rootRegion, firstRegion) < 0);
483    assertTrue(comp.compare(rootRegion, lastRegion) < 0);
484    assertTrue(comp.compare(firstRegion, lastRegion) < 0);
485
486    // first region split into a, b
487    RegionInfo firstRegiona =
488      createRegionInfo(td.getTableName(), HConstants.EMPTY_START_ROW, Bytes.toBytes("aaa"), true);
489    RegionInfo firstRegionb =
490      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), true);
491    // last region split into a, b
492    RegionInfo lastRegiona =
493      createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ddd"), true);
494    RegionInfo lastRegionb =
495      createRegionInfo(td.getTableName(), Bytes.toBytes("ddd"), HConstants.EMPTY_END_ROW, true);
496
497    assertTrue(comp.compare(firstRegiona, firstRegiona) == 0);
498    assertTrue(comp.compare(firstRegionb, firstRegionb) == 0);
499    assertTrue(comp.compare(rootRegion, firstRegiona) < 0);
500    assertTrue(comp.compare(rootRegion, firstRegionb) < 0);
501    assertTrue(comp.compare(firstRegion, firstRegiona) < 0);
502    assertTrue(comp.compare(firstRegion, firstRegionb) < 0);
503    assertTrue(comp.compare(firstRegiona, firstRegionb) < 0);
504
505    assertTrue(comp.compare(lastRegiona, lastRegiona) == 0);
506    assertTrue(comp.compare(lastRegionb, lastRegionb) == 0);
507    assertTrue(comp.compare(rootRegion, lastRegiona) < 0);
508    assertTrue(comp.compare(rootRegion, lastRegionb) < 0);
509    assertTrue(comp.compare(lastRegion, lastRegiona) < 0);
510    assertTrue(comp.compare(lastRegion, lastRegionb) < 0);
511    assertTrue(comp.compare(lastRegiona, lastRegionb) < 0);
512
513    assertTrue(comp.compare(firstRegiona, lastRegiona) < 0);
514    assertTrue(comp.compare(firstRegiona, lastRegionb) < 0);
515    assertTrue(comp.compare(firstRegionb, lastRegiona) < 0);
516    assertTrue(comp.compare(firstRegionb, lastRegionb) < 0);
517
518    RegionInfo lastRegionaa =
519      createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false);
520    RegionInfo lastRegionab =
521      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), false);
522
523    assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0);
524    assertTrue(comp.compare(lastRegiona, lastRegionab) < 0);
525    assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0);
526  }
527
528  @Test
529  public void testArchiveOldRegion() throws Exception {
530    // Create regions.
531    TableDescriptor td = createTableDescriptorForCurrentMethod();
532    RegionInfo parent =
533      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
534    RegionInfo splita =
535      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
536    RegionInfo splitb =
537      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
538
539    // Test that when both daughter regions are in place, that we do not
540    // remove the parent.
541    Result parentMetaRow = createResult(parent, splita, splitb);
542    FileSystem fs = FileSystem.get(HTU.getConfiguration());
543    Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
544    // have to set the root directory since we use it in HFileDisposer to figure out to get to the
545    // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
546    // the single test passes, but when the full suite is run, things get borked).
547    CommonFSUtils.setRootDir(fs.getConf(), rootdir);
548    Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName());
549    Path storedir =
550      HRegionFileSystem.getStoreHomedir(tabledir, parent, td.getColumnFamilies()[0].getName());
551    Path storeArchive = HFileArchiveUtil.getStoreArchivePath(this.masterServices.getConfiguration(),
552      parent, tabledir, td.getColumnFamilies()[0].getName());
553    LOG.debug("Table dir:" + tabledir);
554    LOG.debug("Store dir:" + storedir);
555    LOG.debug("Store archive dir:" + storeArchive);
556
557    // add a couple of store files that we can check for
558    FileStatus[] mockFiles = addMockStoreFiles(2, this.masterServices, storedir);
559    // get the current store files for comparison
560    FileStatus[] storeFiles = fs.listStatus(storedir);
561    int index = 0;
562    for (FileStatus file : storeFiles) {
563      LOG.debug("Have store file:" + file.getPath());
564      assertEquals("Got unexpected store file", mockFiles[index].getPath(),
565        storeFiles[index].getPath());
566      index++;
567    }
568
569    // do the cleaning of the parent
570    assertTrue(CatalogJanitor.cleanParent(masterServices, parent, parentMetaRow));
571    Path parentDir = new Path(tabledir, parent.getEncodedName());
572    // Cleanup procedure runs async. Wait till it done.
573    ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
574    assertTrue(!fs.exists(parentDir));
575    LOG.debug("Finished cleanup of parent region");
576
577    // and now check to make sure that the files have actually been archived
578    FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
579    logFiles("archived files", storeFiles);
580    logFiles("archived files", archivedStoreFiles);
581
582    assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
583
584    // cleanup
585    CommonFSUtils.delete(fs, rootdir, true);
586  }
587
588  /**
589   * @param description description of the files for logging
590   * @param storeFiles  the status of the files to log
591   */
592  private void logFiles(String description, FileStatus[] storeFiles) {
593    LOG.debug("Current " + description + ": ");
594    for (FileStatus file : storeFiles) {
595      LOG.debug(Objects.toString(file.getPath()));
596    }
597  }
598
599  /**
600   * Test that if a store file with the same name is present as those already backed up cause the
601   * already archived files to be timestamped backup
602   */
603  @Test
604  public void testDuplicateHFileResolution() throws Exception {
605    TableDescriptor td = createTableDescriptorForCurrentMethod();
606
607    // Create regions.
608    RegionInfo parent =
609      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
610    RegionInfo splita =
611      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
612    RegionInfo splitb =
613      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
614    // Test that when both daughter regions are in place, that we do not
615    // remove the parent.
616    Result r = createResult(parent, splita, splitb);
617    FileSystem fs = FileSystem.get(HTU.getConfiguration());
618    Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
619    // Have to set the root directory since we use it in HFileDisposer to figure out to get to the
620    // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
621    // the single test passes, but when the full suite is run, things get borked).
622    CommonFSUtils.setRootDir(fs.getConf(), rootdir);
623    Path tabledir = CommonFSUtils.getTableDir(rootdir, parent.getTable());
624    Path storedir =
625      HRegionFileSystem.getStoreHomedir(tabledir, parent, td.getColumnFamilies()[0].getName());
626    LOG.info("Old root:" + rootdir);
627    LOG.info("Old table:" + tabledir);
628    LOG.info("Old store:" + storedir);
629
630    Path storeArchive = HFileArchiveUtil.getStoreArchivePath(this.masterServices.getConfiguration(),
631      parent, tabledir, td.getColumnFamilies()[0].getName());
632    LOG.info("Old archive:" + storeArchive);
633
634    // enable archiving, make sure that files get archived
635    addMockStoreFiles(2, this.masterServices, storedir);
636    // get the current store files for comparison
637    FileStatus[] storeFiles = fs.listStatus(storedir);
638    // Do the cleaning of the parent
639    assertTrue(CatalogJanitor.cleanParent(masterServices, parent, r));
640    Path parentDir = new Path(tabledir, parent.getEncodedName());
641    ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
642    assertTrue(!fs.exists(parentDir));
643
644    // And now check to make sure that the files have actually been archived
645    FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
646    assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
647
648    // now add store files with the same names as before to check backup
649    // enable archiving, make sure that files get archived
650    addMockStoreFiles(2, this.masterServices, storedir);
651
652    // Do the cleaning of the parent
653    assertTrue(CatalogJanitor.cleanParent(masterServices, parent, r));
654    // Cleanup procedure runs async. Wait till it done.
655    ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
656    assertTrue(!fs.exists(parentDir));
657
658    // and now check to make sure that the files have actually been archived
659    archivedStoreFiles = fs.listStatus(storeArchive);
660    assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true);
661  }
662
663  @Test
664  public void testAlreadyRunningStatus() throws Exception {
665    int numberOfThreads = 2;
666    List<Integer> gcValues = new ArrayList<>();
667    Thread[] threads = new Thread[numberOfThreads];
668    for (int i = 0; i < numberOfThreads; i++) {
669      threads[i] = new Thread(() -> {
670        try {
671          gcValues.add(janitor.scan());
672        } catch (IOException e) {
673          throw new RuntimeException(e);
674        }
675      });
676    }
677    for (int i = 0; i < numberOfThreads; i++) {
678      threads[i].start();
679    }
680    for (int i = 0; i < numberOfThreads; i++) {
681      threads[i].join();
682    }
683    assertTrue("One janitor.scan() call should have returned -1", gcValues.contains(-1));
684  }
685
686  private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
687    throws IOException {
688    // get the existing store files
689    FileSystem fs = services.getMasterFileSystem().getFileSystem();
690    fs.mkdirs(storedir);
691    // create the store files in the parent
692    for (int i = 0; i < count; i++) {
693      Path storeFile = new Path(storedir, "_store" + i);
694      FSDataOutputStream dos = fs.create(storeFile, true);
695      dos.writeBytes("Some data: " + i);
696      dos.close();
697    }
698    LOG.debug("Adding " + count + " store files to the storedir:" + storedir);
699    // make sure the mock store files are there
700    FileStatus[] storeFiles = fs.listStatus(storedir);
701    assertEquals("Didn't have expected store files", count, storeFiles.length);
702    return storeFiles;
703  }
704
705  private String setRootDirAndCleanIt(final HBaseTestingUtil htu, final String subdir)
706    throws IOException {
707    Path testdir = htu.getDataTestDir(subdir);
708    FileSystem fs = FileSystem.get(htu.getConfiguration());
709    if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true));
710    CommonFSUtils.setRootDir(htu.getConfiguration(), testdir);
711    return CommonFSUtils.getRootDir(htu.getConfiguration()).toString();
712  }
713
714  private Path createReferences(final MasterServices services, final TableDescriptor td,
715    final RegionInfo parent, final RegionInfo daughter, final byte[] midkey, final boolean top)
716    throws IOException {
717    Path rootdir = services.getMasterFileSystem().getRootDir();
718    Path tabledir = CommonFSUtils.getTableDir(rootdir, parent.getTable());
719    Path storedir =
720      HRegionFileSystem.getStoreHomedir(tabledir, daughter, td.getColumnFamilies()[0].getName());
721    Reference ref =
722      top ? Reference.createTopReference(midkey) : Reference.createBottomReference(midkey);
723    long now = EnvironmentEdgeManager.currentTime();
724    // Reference name has this format: StoreFile#REF_NAME_PARSER
725    Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
726    FileSystem fs = services.getMasterFileSystem().getFileSystem();
727    ref.write(fs, p);
728    return p;
729  }
730
731  private Result createResult(final RegionInfo parent, final RegionInfo a, final RegionInfo b)
732    throws IOException {
733    return MetaMockingUtil.getMetaTableRowResult(parent, null, a, b);
734  }
735}