001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.janitor;
019
020import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024import static org.mockito.ArgumentMatchers.any;
025import static org.mockito.Mockito.doAnswer;
026import static org.mockito.Mockito.doReturn;
027import static org.mockito.Mockito.doThrow;
028import static org.mockito.Mockito.spy;
029import static org.mockito.Mockito.when;
030
031import java.io.IOException;
032import java.util.ArrayList;
033import java.util.List;
034import java.util.Map;
035import java.util.Objects;
036import java.util.SortedMap;
037import java.util.TreeMap;
038import java.util.concurrent.atomic.AtomicBoolean;
039import org.apache.hadoop.fs.FSDataOutputStream;
040import org.apache.hadoop.fs.FileStatus;
041import org.apache.hadoop.fs.FileSystem;
042import org.apache.hadoop.fs.Path;
043import org.apache.hadoop.hbase.HBaseClassTestRule;
044import org.apache.hadoop.hbase.HBaseTestingUtil;
045import org.apache.hadoop.hbase.HConstants;
046import org.apache.hadoop.hbase.MetaMockingUtil;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
049import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
050import org.apache.hadoop.hbase.client.RegionInfo;
051import org.apache.hadoop.hbase.client.RegionInfoBuilder;
052import org.apache.hadoop.hbase.client.Result;
053import org.apache.hadoop.hbase.client.TableDescriptor;
054import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
055import org.apache.hadoop.hbase.io.Reference;
056import org.apache.hadoop.hbase.master.MasterFileSystem;
057import org.apache.hadoop.hbase.master.MasterServices;
058import org.apache.hadoop.hbase.master.assignment.MockMasterServices;
059import org.apache.hadoop.hbase.master.janitor.CatalogJanitor.SplitParentFirstComparator;
060import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
061import org.apache.hadoop.hbase.regionserver.ChunkCreator;
062import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
063import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
064import org.apache.hadoop.hbase.testclassification.MasterTests;
065import org.apache.hadoop.hbase.testclassification.MediumTests;
066import org.apache.hadoop.hbase.util.Bytes;
067import org.apache.hadoop.hbase.util.CommonFSUtils;
068import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
069import org.apache.hadoop.hbase.util.HFileArchiveUtil;
070import org.junit.After;
071import org.junit.Before;
072import org.junit.BeforeClass;
073import org.junit.ClassRule;
074import org.junit.Rule;
075import org.junit.Test;
076import org.junit.experimental.categories.Category;
077import org.junit.rules.TestName;
078import org.slf4j.Logger;
079import org.slf4j.LoggerFactory;
080
081@Category({ MasterTests.class, MediumTests.class })
082public class TestCatalogJanitor {
083
084  @ClassRule
085  public static final HBaseClassTestRule CLASS_RULE =
086    HBaseClassTestRule.forClass(TestCatalogJanitor.class);
087
088  private static final Logger LOG = LoggerFactory.getLogger(TestCatalogJanitor.class);
089
090  private static final HBaseTestingUtil HTU = new HBaseTestingUtil();
091
092  @Rule
093  public final TestName name = new TestName();
094
095  private MockMasterServices masterServices;
096  private CatalogJanitor janitor;
097
098  @BeforeClass
099  public static void beforeClass() throws Exception {
100    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
101      MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
102  }
103
104  @Before
105  public void setup() throws Exception {
106    setRootDirAndCleanIt(HTU, this.name.getMethodName());
107    this.masterServices = new MockMasterServices(HTU.getConfiguration());
108    this.masterServices.start(10, null);
109    this.janitor = new CatalogJanitor(masterServices);
110  }
111
112  @After
113  public void teardown() {
114    this.janitor.shutdown(true);
115    this.masterServices.stop("DONE");
116  }
117
118  private RegionInfo createRegionInfo(TableName tableName, byte[] startKey, byte[] endKey) {
119    return createRegionInfo(tableName, startKey, endKey, false);
120  }
121
122  private RegionInfo createRegionInfo(TableName tableName, byte[] startKey, byte[] endKey,
123    boolean split) {
124    return RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(endKey)
125      .setSplit(split).build();
126  }
127
128  @Test
129  public void testCleanMerge() throws IOException {
130    TableDescriptor td = createTableDescriptorForCurrentMethod();
131    // Create regions.
132    RegionInfo merged =
133      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
134    RegionInfo parenta =
135      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
136    RegionInfo parentb =
137      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
138
139    List<RegionInfo> parents = new ArrayList<>();
140    parents.add(parenta);
141    parents.add(parentb);
142
143    Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
144    Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName());
145    Path storedir =
146      HRegionFileSystem.getStoreHomedir(tabledir, merged, td.getColumnFamilies()[0].getName());
147
148    Path parentaRef = createMergeReferenceFile(storedir, merged, parenta);
149    Path parentbRef = createMergeReferenceFile(storedir, merged, parentb);
150
151    // references exist, should not clean
152    assertFalse(CatalogJanitor.cleanMergeRegion(masterServices, merged, parents));
153
154    masterServices.getMasterFileSystem().getFileSystem().delete(parentaRef, false);
155
156    // one reference still exists, should not clean
157    assertFalse(CatalogJanitor.cleanMergeRegion(masterServices, merged, parents));
158
159    masterServices.getMasterFileSystem().getFileSystem().delete(parentbRef, false);
160
161    // all references removed, should clean
162    assertTrue(CatalogJanitor.cleanMergeRegion(masterServices, merged, parents));
163  }
164
165  @Test
166  public void testDontCleanMergeIfFileSystemException() throws IOException {
167    TableDescriptor td = createTableDescriptorForCurrentMethod();
168    // Create regions.
169    RegionInfo merged =
170      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
171    RegionInfo parenta =
172      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
173    RegionInfo parentb =
174      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
175
176    List<RegionInfo> parents = new ArrayList<>();
177    parents.add(parenta);
178    parents.add(parentb);
179
180    Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
181    Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName());
182    Path storedir =
183      HRegionFileSystem.getStoreHomedir(tabledir, merged, td.getColumnFamilies()[0].getName());
184    createMergeReferenceFile(storedir, merged, parenta);
185
186    MasterServices mockedMasterServices = spy(masterServices);
187    MasterFileSystem mockedMasterFileSystem = spy(masterServices.getMasterFileSystem());
188    FileSystem mockedFileSystem = spy(masterServices.getMasterFileSystem().getFileSystem());
189
190    when(mockedMasterServices.getMasterFileSystem()).thenReturn(mockedMasterFileSystem);
191    when(mockedMasterFileSystem.getFileSystem()).thenReturn(mockedFileSystem);
192
193    // throw on the first exists check
194    doThrow(new IOException("Some exception")).when(mockedFileSystem).exists(any());
195
196    assertFalse(CatalogJanitor.cleanMergeRegion(mockedMasterServices, merged, parents));
197
198    // throw on the second exists check (within HRegionfileSystem)
199    AtomicBoolean returned = new AtomicBoolean(false);
200    doAnswer(invocationOnMock -> {
201      if (!returned.get()) {
202        returned.set(true);
203        return masterServices.getMasterFileSystem().getFileSystem()
204          .exists(invocationOnMock.getArgument(0));
205      }
206      throw new IOException("Some exception");
207    }).when(mockedFileSystem).exists(any());
208
209    assertFalse(CatalogJanitor.cleanMergeRegion(mockedMasterServices, merged, parents));
210  }
211
212  private Path createMergeReferenceFile(Path storeDir, RegionInfo mergedRegion,
213    RegionInfo parentRegion) throws IOException {
214    Reference ref = Reference.createTopReference(mergedRegion.getStartKey());
215    long now = EnvironmentEdgeManager.currentTime();
216    // Reference name has this format: StoreFile#REF_NAME_PARSER
217    Path p = new Path(storeDir, Long.toString(now) + "." + parentRegion.getEncodedName());
218    FileSystem fs = this.masterServices.getMasterFileSystem().getFileSystem();
219    return ref.write(fs, p);
220  }
221
222  /**
223   * Test clearing a split parent.
224   */
225  @Test
226  public void testCleanParent() throws IOException, InterruptedException {
227    TableDescriptor td = createTableDescriptorForCurrentMethod();
228    // Create regions.
229    RegionInfo parent =
230      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
231    RegionInfo splita =
232      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
233    RegionInfo splitb =
234      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
235    // Test that when both daughter regions are in place, that we do not remove the parent.
236    Result r = createResult(parent, splita, splitb);
237    // Add a reference under splitA directory so we don't clear out the parent.
238    Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
239    Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName());
240    Path parentdir = new Path(tabledir, parent.getEncodedName());
241    Path storedir =
242      HRegionFileSystem.getStoreHomedir(tabledir, splita, td.getColumnFamilies()[0].getName());
243    Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
244    long now = EnvironmentEdgeManager.currentTime();
245    // Reference name has this format: StoreFile#REF_NAME_PARSER
246    Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
247    FileSystem fs = this.masterServices.getMasterFileSystem().getFileSystem();
248    Path path = ref.write(fs, p);
249    assertTrue(fs.exists(path));
250    LOG.info("Created reference " + path);
251    // Add a parentdir for kicks so can check it gets removed by the catalogjanitor.
252    fs.mkdirs(parentdir);
253    assertFalse(CatalogJanitor.cleanParent(masterServices, parent, r));
254    ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
255    assertTrue(fs.exists(parentdir));
256    // Remove the reference file and try again.
257    assertTrue(fs.delete(p, true));
258    assertTrue(CatalogJanitor.cleanParent(masterServices, parent, r));
259    // Parent cleanup is run async as a procedure. Make sure parentdir is removed.
260    ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
261    assertTrue(!fs.exists(parentdir));
262  }
263
264  /**
265   * Make sure parent gets cleaned up even if daughter is cleaned up before it.
266   */
267  @Test
268  public void testParentCleanedEvenIfDaughterGoneFirst() throws IOException, InterruptedException {
269    parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(this.name.getMethodName(),
270      Bytes.toBytes("eee"));
271  }
272
273  /**
274   * Make sure last parent with empty end key gets cleaned up even if daughter is cleaned up before
275   * it.
276   */
277  @Test
278  public void testLastParentCleanedEvenIfDaughterGoneFirst()
279    throws IOException, InterruptedException {
280    parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(this.name.getMethodName(), new byte[0]);
281  }
282
283  /**
284   * @return A TableDescriptor with a tableName of current method name and a column family that is
285   *         MockMasterServices.DEFAULT_COLUMN_FAMILY_NAME)
286   */
287  private TableDescriptor createTableDescriptorForCurrentMethod() {
288    ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
289      .newBuilder(Bytes.toBytes(MockMasterServices.DEFAULT_COLUMN_FAMILY_NAME)).build();
290    return TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name.getMethodName()))
291      .setColumnFamily(columnFamilyDescriptor).build();
292  }
293
294  /**
295   * Make sure parent with specified end key gets cleaned up even if daughter is cleaned up before
296   * it.
297   * @param rootDir    the test case name, used as the HBase testing utility root
298   * @param lastEndKey the end key of the split parent
299   */
300  private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(final String rootDir,
301    final byte[] lastEndKey) throws IOException, InterruptedException {
302    TableDescriptor td = createTableDescriptorForCurrentMethod();
303    // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
304    RegionInfo parent = createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), lastEndKey);
305    // Sleep a second else the encoded name on these regions comes out
306    // same for all with same start key and made in same second.
307    Thread.sleep(1001);
308
309    // Daughter a
310    RegionInfo splita =
311      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
312    Thread.sleep(1001);
313    // Make daughters of daughter a; splitaa and splitab.
314    RegionInfo splitaa =
315      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"));
316    RegionInfo splitab =
317      createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"));
318
319    // Daughter b
320    RegionInfo splitb = createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), lastEndKey);
321    Thread.sleep(1001);
322    // Make Daughters of daughterb; splitba and splitbb.
323    RegionInfo splitba =
324      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("ddd"));
325    RegionInfo splitbb = createRegionInfo(td.getTableName(), Bytes.toBytes("ddd"), lastEndKey);
326
327    // First test that our Comparator works right up in CatalogJanitor.
328    SortedMap<RegionInfo, Result> regions =
329      new TreeMap<>(new CatalogJanitor.SplitParentFirstComparator());
330    // Now make sure that this regions map sorts as we expect it to.
331    regions.put(parent, createResult(parent, splita, splitb));
332    regions.put(splitb, createResult(splitb, splitba, splitbb));
333    regions.put(splita, createResult(splita, splitaa, splitab));
334    // Assert its properly sorted.
335    int index = 0;
336    for (Map.Entry<RegionInfo, Result> e : regions.entrySet()) {
337      if (index == 0) {
338        assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName()));
339      } else if (index == 1) {
340        assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName()));
341      } else if (index == 2) {
342        assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName()));
343      }
344      index++;
345    }
346
347    // Now play around with the cleanParent function. Create a ref from splita up to the parent.
348    Path splitaRef =
349      createReferences(this.masterServices, td, parent, splita, Bytes.toBytes("ccc"), false);
350    // Make sure actual super parent sticks around because splita has a ref.
351    assertFalse(CatalogJanitor.cleanParent(masterServices, parent, regions.get(parent)));
352
353    // splitba, and split bb, do not have dirs in fs. That means that if
354    // we test splitb, it should get cleaned up.
355    assertTrue(CatalogJanitor.cleanParent(masterServices, splitb, regions.get(splitb)));
356
357    // Now remove ref from splita to parent... so parent can be let go and so
358    // the daughter splita can be split (can't split if still references).
359    // BUT make the timing such that the daughter gets cleaned up before we
360    // can get a chance to let go of the parent.
361    FileSystem fs = FileSystem.get(HTU.getConfiguration());
362    assertTrue(fs.delete(splitaRef, true));
363    // Create the refs from daughters of splita.
364    Path splitaaRef =
365      createReferences(this.masterServices, td, splita, splitaa, Bytes.toBytes("bbb"), false);
366    Path splitabRef =
367      createReferences(this.masterServices, td, splita, splitab, Bytes.toBytes("bbb"), true);
368
369    // Test splita. It should stick around because references from splitab, etc.
370    assertFalse(CatalogJanitor.cleanParent(masterServices, splita, regions.get(splita)));
371
372    // Now clean up parent daughter first. Remove references from its daughters.
373    assertTrue(fs.delete(splitaaRef, true));
374    assertTrue(fs.delete(splitabRef, true));
375    assertTrue(CatalogJanitor.cleanParent(masterServices, splita, regions.get(splita)));
376
377    // Super parent should get cleaned up now both splita and splitb are gone.
378    assertTrue(CatalogJanitor.cleanParent(masterServices, parent, regions.get(parent)));
379  }
380
381  /**
382   * CatalogJanitor.scan() should not clean parent regions if their own parents are still
383   * referencing them. This ensures that grandparent regions do not point to deleted parent regions.
384   */
385  @Test
386  public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
387    TableDescriptor td = createTableDescriptorForCurrentMethod();
388    // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
389
390    // Parent
391    RegionInfo parent =
392      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), HConstants.EMPTY_BYTE_ARRAY, true);
393    // Sleep a second else the encoded name on these regions comes out
394    // same for all with same start key and made in same second.
395    Thread.sleep(1001);
396
397    // Daughter a
398    RegionInfo splita =
399      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"), true);
400    Thread.sleep(1001);
401
402    // Make daughters of daughter a; splitaa and splitab.
403    RegionInfo splitaa =
404      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), false);
405    RegionInfo splitab =
406      createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false);
407
408    // Daughter b
409    RegionInfo splitb =
410      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), HConstants.EMPTY_BYTE_ARRAY);
411    Thread.sleep(1001);
412
413    // Parent has daughters splita and splitb. Splita has daughters splitaa and splitab.
414    final Map<RegionInfo, Result> splitParents = new TreeMap<>(new SplitParentFirstComparator());
415    splitParents.put(parent, createResult(parent, splita, splitb));
416    // simulate that splita goes offline when it is split
417    splita = RegionInfoBuilder.newBuilder(splita).setOffline(true).build();
418    splitParents.put(splita, createResult(splita, splitaa, splitab));
419
420    final Map<RegionInfo, Result> mergedRegions = new TreeMap<>();
421    CatalogJanitor spy = spy(this.janitor);
422
423    CatalogJanitorReport report = new CatalogJanitorReport();
424    report.count = 10;
425    report.mergedRegions.putAll(mergedRegions);
426    report.splitParents.putAll(splitParents);
427
428    doReturn(report).when(spy).scanForReport();
429
430    // Create ref from splita to parent
431    LOG.info("parent=" + parent.getShortNameToLog() + ", splita=" + splita.getShortNameToLog());
432    Path splitaRef =
433      createReferences(this.masterServices, td, parent, splita, Bytes.toBytes("ccc"), false);
434    LOG.info("Created reference " + splitaRef);
435
436    // Parent and splita should not be removed because a reference from splita to parent.
437    int gcs = spy.scan();
438    assertEquals(0, gcs);
439
440    // Now delete the ref
441    FileSystem fs = FileSystem.get(HTU.getConfiguration());
442    assertTrue(fs.delete(splitaRef, true));
443
444    // now, both parent, and splita can be deleted
445    gcs = spy.scan();
446    assertEquals(2, gcs);
447  }
448
449  /**
450   * Test that we correctly archive all the storefiles when a region is deleted
451   */
452  @Test
453  public void testSplitParentFirstComparator() {
454    SplitParentFirstComparator comp = new SplitParentFirstComparator();
455    TableDescriptor td = createTableDescriptorForCurrentMethod();
456
457    /*
458     * Region splits: rootRegion --- firstRegion --- firstRegiona | |- firstRegionb | |- lastRegion
459     * --- lastRegiona --- lastRegionaa | |- lastRegionab |- lastRegionb rootRegion : [] - []
460     * firstRegion : [] - bbb lastRegion : bbb - [] firstRegiona : [] - aaa firstRegionb : aaa - bbb
461     * lastRegiona : bbb - ddd lastRegionb : ddd - []
462     */
463
464    // root region
465    RegionInfo rootRegion = createRegionInfo(td.getTableName(), HConstants.EMPTY_START_ROW,
466      HConstants.EMPTY_END_ROW, true);
467    RegionInfo firstRegion =
468      createRegionInfo(td.getTableName(), HConstants.EMPTY_START_ROW, Bytes.toBytes("bbb"), true);
469    RegionInfo lastRegion =
470      createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), HConstants.EMPTY_END_ROW, true);
471
472    assertTrue(comp.compare(rootRegion, rootRegion) == 0);
473    assertTrue(comp.compare(firstRegion, firstRegion) == 0);
474    assertTrue(comp.compare(lastRegion, lastRegion) == 0);
475    assertTrue(comp.compare(rootRegion, firstRegion) < 0);
476    assertTrue(comp.compare(rootRegion, lastRegion) < 0);
477    assertTrue(comp.compare(firstRegion, lastRegion) < 0);
478
479    // first region split into a, b
480    RegionInfo firstRegiona =
481      createRegionInfo(td.getTableName(), HConstants.EMPTY_START_ROW, Bytes.toBytes("aaa"), true);
482    RegionInfo firstRegionb =
483      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), true);
484    // last region split into a, b
485    RegionInfo lastRegiona =
486      createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ddd"), true);
487    RegionInfo lastRegionb =
488      createRegionInfo(td.getTableName(), Bytes.toBytes("ddd"), HConstants.EMPTY_END_ROW, true);
489
490    assertTrue(comp.compare(firstRegiona, firstRegiona) == 0);
491    assertTrue(comp.compare(firstRegionb, firstRegionb) == 0);
492    assertTrue(comp.compare(rootRegion, firstRegiona) < 0);
493    assertTrue(comp.compare(rootRegion, firstRegionb) < 0);
494    assertTrue(comp.compare(firstRegion, firstRegiona) < 0);
495    assertTrue(comp.compare(firstRegion, firstRegionb) < 0);
496    assertTrue(comp.compare(firstRegiona, firstRegionb) < 0);
497
498    assertTrue(comp.compare(lastRegiona, lastRegiona) == 0);
499    assertTrue(comp.compare(lastRegionb, lastRegionb) == 0);
500    assertTrue(comp.compare(rootRegion, lastRegiona) < 0);
501    assertTrue(comp.compare(rootRegion, lastRegionb) < 0);
502    assertTrue(comp.compare(lastRegion, lastRegiona) < 0);
503    assertTrue(comp.compare(lastRegion, lastRegionb) < 0);
504    assertTrue(comp.compare(lastRegiona, lastRegionb) < 0);
505
506    assertTrue(comp.compare(firstRegiona, lastRegiona) < 0);
507    assertTrue(comp.compare(firstRegiona, lastRegionb) < 0);
508    assertTrue(comp.compare(firstRegionb, lastRegiona) < 0);
509    assertTrue(comp.compare(firstRegionb, lastRegionb) < 0);
510
511    RegionInfo lastRegionaa =
512      createRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false);
513    RegionInfo lastRegionab =
514      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), false);
515
516    assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0);
517    assertTrue(comp.compare(lastRegiona, lastRegionab) < 0);
518    assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0);
519  }
520
521  @Test
522  public void testArchiveOldRegion() throws Exception {
523    // Create regions.
524    TableDescriptor td = createTableDescriptorForCurrentMethod();
525    RegionInfo parent =
526      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
527    RegionInfo splita =
528      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
529    RegionInfo splitb =
530      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
531
532    // Test that when both daughter regions are in place, that we do not
533    // remove the parent.
534    Result parentMetaRow = createResult(parent, splita, splitb);
535    FileSystem fs = FileSystem.get(HTU.getConfiguration());
536    Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
537    // have to set the root directory since we use it in HFileDisposer to figure out to get to the
538    // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
539    // the single test passes, but when the full suite is run, things get borked).
540    CommonFSUtils.setRootDir(fs.getConf(), rootdir);
541    Path tabledir = CommonFSUtils.getTableDir(rootdir, td.getTableName());
542    Path storedir =
543      HRegionFileSystem.getStoreHomedir(tabledir, parent, td.getColumnFamilies()[0].getName());
544    Path storeArchive = HFileArchiveUtil.getStoreArchivePath(this.masterServices.getConfiguration(),
545      parent, tabledir, td.getColumnFamilies()[0].getName());
546    LOG.debug("Table dir:" + tabledir);
547    LOG.debug("Store dir:" + storedir);
548    LOG.debug("Store archive dir:" + storeArchive);
549
550    // add a couple of store files that we can check for
551    FileStatus[] mockFiles = addMockStoreFiles(2, this.masterServices, storedir);
552    // get the current store files for comparison
553    FileStatus[] storeFiles = fs.listStatus(storedir);
554    int index = 0;
555    for (FileStatus file : storeFiles) {
556      LOG.debug("Have store file:" + file.getPath());
557      assertEquals("Got unexpected store file", mockFiles[index].getPath(),
558        storeFiles[index].getPath());
559      index++;
560    }
561
562    // do the cleaning of the parent
563    assertTrue(CatalogJanitor.cleanParent(masterServices, parent, parentMetaRow));
564    Path parentDir = new Path(tabledir, parent.getEncodedName());
565    // Cleanup procedure runs async. Wait till it done.
566    ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
567    assertTrue(!fs.exists(parentDir));
568    LOG.debug("Finished cleanup of parent region");
569
570    // and now check to make sure that the files have actually been archived
571    FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
572    logFiles("archived files", storeFiles);
573    logFiles("archived files", archivedStoreFiles);
574
575    assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
576
577    // cleanup
578    CommonFSUtils.delete(fs, rootdir, true);
579  }
580
581  /**
582   * @param description description of the files for logging
583   * @param storeFiles  the status of the files to log
584   */
585  private void logFiles(String description, FileStatus[] storeFiles) {
586    LOG.debug("Current " + description + ": ");
587    for (FileStatus file : storeFiles) {
588      LOG.debug(Objects.toString(file.getPath()));
589    }
590  }
591
592  /**
593   * Test that if a store file with the same name is present as those already backed up cause the
594   * already archived files to be timestamped backup
595   */
596  @Test
597  public void testDuplicateHFileResolution() throws Exception {
598    TableDescriptor td = createTableDescriptorForCurrentMethod();
599
600    // Create regions.
601    RegionInfo parent =
602      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
603    RegionInfo splita =
604      createRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
605    RegionInfo splitb =
606      createRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
607    // Test that when both daughter regions are in place, that we do not
608    // remove the parent.
609    Result r = createResult(parent, splita, splitb);
610    FileSystem fs = FileSystem.get(HTU.getConfiguration());
611    Path rootdir = this.masterServices.getMasterFileSystem().getRootDir();
612    // Have to set the root directory since we use it in HFileDisposer to figure out to get to the
613    // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
614    // the single test passes, but when the full suite is run, things get borked).
615    CommonFSUtils.setRootDir(fs.getConf(), rootdir);
616    Path tabledir = CommonFSUtils.getTableDir(rootdir, parent.getTable());
617    Path storedir =
618      HRegionFileSystem.getStoreHomedir(tabledir, parent, td.getColumnFamilies()[0].getName());
619    LOG.info("Old root:" + rootdir);
620    LOG.info("Old table:" + tabledir);
621    LOG.info("Old store:" + storedir);
622
623    Path storeArchive = HFileArchiveUtil.getStoreArchivePath(this.masterServices.getConfiguration(),
624      parent, tabledir, td.getColumnFamilies()[0].getName());
625    LOG.info("Old archive:" + storeArchive);
626
627    // enable archiving, make sure that files get archived
628    addMockStoreFiles(2, this.masterServices, storedir);
629    // get the current store files for comparison
630    FileStatus[] storeFiles = fs.listStatus(storedir);
631    // Do the cleaning of the parent
632    assertTrue(CatalogJanitor.cleanParent(masterServices, parent, r));
633    Path parentDir = new Path(tabledir, parent.getEncodedName());
634    ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
635    assertTrue(!fs.exists(parentDir));
636
637    // And now check to make sure that the files have actually been archived
638    FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
639    assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
640
641    // now add store files with the same names as before to check backup
642    // enable archiving, make sure that files get archived
643    addMockStoreFiles(2, this.masterServices, storedir);
644
645    // Do the cleaning of the parent
646    assertTrue(CatalogJanitor.cleanParent(masterServices, parent, r));
647    // Cleanup procedure runs async. Wait till it done.
648    ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor());
649    assertTrue(!fs.exists(parentDir));
650
651    // and now check to make sure that the files have actually been archived
652    archivedStoreFiles = fs.listStatus(storeArchive);
653    assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true);
654  }
655
656  @Test
657  public void testAlreadyRunningStatus() throws Exception {
658    int numberOfThreads = 2;
659    List<Integer> gcValues = new ArrayList<>();
660    Thread[] threads = new Thread[numberOfThreads];
661    for (int i = 0; i < numberOfThreads; i++) {
662      threads[i] = new Thread(() -> {
663        try {
664          gcValues.add(janitor.scan());
665        } catch (IOException e) {
666          throw new RuntimeException(e);
667        }
668      });
669    }
670    for (int i = 0; i < numberOfThreads; i++) {
671      threads[i].start();
672    }
673    for (int i = 0; i < numberOfThreads; i++) {
674      threads[i].join();
675    }
676    assertTrue("One janitor.scan() call should have returned -1", gcValues.contains(-1));
677  }
678
679  private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
680    throws IOException {
681    // get the existing store files
682    FileSystem fs = services.getMasterFileSystem().getFileSystem();
683    fs.mkdirs(storedir);
684    // create the store files in the parent
685    for (int i = 0; i < count; i++) {
686      Path storeFile = new Path(storedir, "_store" + i);
687      FSDataOutputStream dos = fs.create(storeFile, true);
688      dos.writeBytes("Some data: " + i);
689      dos.close();
690    }
691    LOG.debug("Adding " + count + " store files to the storedir:" + storedir);
692    // make sure the mock store files are there
693    FileStatus[] storeFiles = fs.listStatus(storedir);
694    assertEquals("Didn't have expected store files", count, storeFiles.length);
695    return storeFiles;
696  }
697
698  private String setRootDirAndCleanIt(final HBaseTestingUtil htu, final String subdir)
699    throws IOException {
700    Path testdir = htu.getDataTestDir(subdir);
701    FileSystem fs = FileSystem.get(htu.getConfiguration());
702    if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true));
703    CommonFSUtils.setRootDir(htu.getConfiguration(), testdir);
704    return CommonFSUtils.getRootDir(htu.getConfiguration()).toString();
705  }
706
707  private Path createReferences(final MasterServices services, final TableDescriptor td,
708    final RegionInfo parent, final RegionInfo daughter, final byte[] midkey, final boolean top)
709    throws IOException {
710    Path rootdir = services.getMasterFileSystem().getRootDir();
711    Path tabledir = CommonFSUtils.getTableDir(rootdir, parent.getTable());
712    Path storedir =
713      HRegionFileSystem.getStoreHomedir(tabledir, daughter, td.getColumnFamilies()[0].getName());
714    Reference ref =
715      top ? Reference.createTopReference(midkey) : Reference.createBottomReference(midkey);
716    long now = EnvironmentEdgeManager.currentTime();
717    // Reference name has this format: StoreFile#REF_NAME_PARSER
718    Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
719    FileSystem fs = services.getMasterFileSystem().getFileSystem();
720    ref.write(fs, p);
721    return p;
722  }
723
724  private Result createResult(final RegionInfo parent, final RegionInfo a, final RegionInfo b)
725    throws IOException {
726    return MetaMockingUtil.getMetaTableRowResult(parent, null, a, b);
727  }
728}