001/*
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019package org.apache.hadoop.hbase.master;
020
021import java.io.IOException;
022import java.io.InputStream;
023import java.util.ArrayList;
024import java.util.Comparator;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Map;
028import java.util.Properties;
029import java.util.TreeMap;
030import java.util.concurrent.atomic.AtomicBoolean;
031import java.util.stream.Collectors;
032
033import org.apache.hadoop.conf.Configuration;
034import org.apache.hadoop.fs.FileSystem;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.hbase.HBaseConfiguration;
037import org.apache.hadoop.hbase.HConstants;
038import org.apache.hadoop.hbase.HRegionLocation;
039import org.apache.hadoop.hbase.MetaTableAccessor;
040import org.apache.hadoop.hbase.RegionLocations;
041import org.apache.hadoop.hbase.ScheduledChore;
042import org.apache.hadoop.hbase.ServerName;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
045import org.apache.hadoop.hbase.client.Connection;
046import org.apache.hadoop.hbase.client.ConnectionFactory;
047import org.apache.hadoop.hbase.client.Get;
048import org.apache.hadoop.hbase.client.Put;
049import org.apache.hadoop.hbase.client.RegionInfo;
050import org.apache.hadoop.hbase.client.Result;
051import org.apache.hadoop.hbase.client.Table;
052import org.apache.hadoop.hbase.client.TableDescriptor;
053import org.apache.hadoop.hbase.client.TableState;
054import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
055import org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure;
056import org.apache.hadoop.hbase.master.assignment.GCRegionProcedure;
057import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
058import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
059import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
060import org.apache.hadoop.hbase.util.Bytes;
061import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
062import org.apache.hadoop.hbase.util.FSUtils;
063import org.apache.hadoop.hbase.util.Pair;
064import org.apache.hadoop.hbase.util.PairOfSameType;
065import org.apache.hadoop.hbase.util.Threads;
066import org.apache.yetus.audience.InterfaceAudience;
067import org.slf4j.Logger;
068import org.slf4j.LoggerFactory;
069
070import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
071
072/**
073 * A janitor for the catalog tables. Scans the <code>hbase:meta</code> catalog
074 * table on a period. Makes a lastReport on state of hbase:meta. Looks for unused
075 * regions to garbage collect. Scan of hbase:meta runs if we are NOT in maintenance
076 * mode, if we are NOT shutting down, AND if the assignmentmanager is loaded.
077 * Playing it safe, we will garbage collect no-longer needed region references
078 * only if there are no regions-in-transition (RIT).
079 */
080// TODO: Only works with single hbase:meta region currently.  Fix.
081// TODO: Should it start over every time? Could it continue if runs into problem? Only if
082// problem does not mess up 'results'.
083@InterfaceAudience.Private
084public class CatalogJanitor extends ScheduledChore {
085  private static final Logger LOG = LoggerFactory.getLogger(CatalogJanitor.class.getName());
086  private final AtomicBoolean alreadyRunning = new AtomicBoolean(false);
087  private final AtomicBoolean enabled = new AtomicBoolean(true);
088  private final MasterServices services;
089
090  /**
091   * Saved report from last hbase:meta scan to completion. May be stale if having trouble
092   * completing scan. Check its date.
093   */
094  private volatile Report lastReport;
095
096  CatalogJanitor(final MasterServices services) {
097    super("CatalogJanitor-" + services.getServerName().toShortString(), services,
098      services.getConfiguration().getInt("hbase.catalogjanitor.interval", 300000));
099    this.services = services;
100  }
101
102  @Override
103  protected boolean initialChore() {
104    try {
105      if (getEnabled()) {
106        scan();
107      }
108    } catch (IOException e) {
109      LOG.warn("Failed initial janitorial scan of hbase:meta table", e);
110      return false;
111    }
112    return true;
113  }
114
115  boolean setEnabled(final boolean enabled) {
116    boolean alreadyEnabled = this.enabled.getAndSet(enabled);
117    // If disabling is requested on an already enabled chore, we could have an active
118    // scan still going on, callers might not be aware of that and do further action thinkng
119    // that no action would be from this chore.  In this case, the right action is to wait for
120    // the active scan to complete before exiting this function.
121    if (!enabled && alreadyEnabled) {
122      while (alreadyRunning.get()) {
123        Threads.sleepWithoutInterrupt(100);
124      }
125    }
126    return alreadyEnabled;
127  }
128
129  boolean getEnabled() {
130    return this.enabled.get();
131  }
132
133  @Override
134  protected void chore() {
135    try {
136      AssignmentManager am = this.services.getAssignmentManager();
137      if (getEnabled() && !this.services.isInMaintenanceMode() &&
138          !this.services.getServerManager().isClusterShutdown() &&
139          isMetaLoaded(am)) {
140        scan();
141      } else {
142        LOG.warn("CatalogJanitor is disabled! Enabled=" + getEnabled() + 
143          ", maintenanceMode=" + this.services.isInMaintenanceMode() + ", am=" + am +
144          ", metaLoaded=" + isMetaLoaded(am) + ", hasRIT=" + isRIT(am) +
145          " clusterShutDown=" + this.services.getServerManager().isClusterShutdown());
146      }
147    } catch (IOException e) {
148      LOG.warn("Failed janitorial scan of hbase:meta table", e);
149    }
150  }
151
152  private static boolean isMetaLoaded(AssignmentManager am) {
153    return am != null && am.isMetaLoaded();
154  }
155
156  private static boolean isRIT(AssignmentManager am) {
157    return isMetaLoaded(am) && am.hasRegionsInTransition();
158  }
159
160  /**
161   * Run janitorial scan of catalog <code>hbase:meta</code> table looking for
162   * garbage to collect.
163   * @return How many items gc'd whether for merge or split.
164   */
165  int scan() throws IOException {
166    int gcs = 0;
167    try {
168      if (!alreadyRunning.compareAndSet(false, true)) {
169        LOG.debug("CatalogJanitor already running");
170        return gcs;
171      }
172      Report report = scanForReport();
173      this.lastReport = report;
174      if (!report.isEmpty()) {
175        LOG.warn(report.toString());
176      }
177
178      if (isRIT(this.services.getAssignmentManager())) {
179        LOG.warn("Playing-it-safe skipping merge/split gc'ing of regions from hbase:meta while " +
180            "regions-in-transition (RIT)");
181      }
182      Map<RegionInfo, Result> mergedRegions = report.mergedRegions;
183      for (Map.Entry<RegionInfo, Result> e : mergedRegions.entrySet()) {
184        if (this.services.isInMaintenanceMode()) {
185          // Stop cleaning if the master is in maintenance mode
186          break;
187        }
188
189        List<RegionInfo> parents = MetaTableAccessor.getMergeRegions(e.getValue().rawCells());
190        if (parents != null && cleanMergeRegion(e.getKey(), parents)) {
191          gcs++;
192        }
193      }
194      // Clean split parents
195      Map<RegionInfo, Result> splitParents = report.splitParents;
196
197      // Now work on our list of found parents. See if any we can clean up.
198      HashSet<String> parentNotCleaned = new HashSet<>();
199      for (Map.Entry<RegionInfo, Result> e : splitParents.entrySet()) {
200        if (this.services.isInMaintenanceMode()) {
201          // Stop cleaning if the master is in maintenance mode
202          break;
203        }
204
205        if (!parentNotCleaned.contains(e.getKey().getEncodedName()) &&
206            cleanParent(e.getKey(), e.getValue())) {
207          gcs++;
208        } else {
209          // We could not clean the parent, so it's daughters should not be
210          // cleaned either (HBASE-6160)
211          PairOfSameType<RegionInfo> daughters =
212              MetaTableAccessor.getDaughterRegions(e.getValue());
213          parentNotCleaned.add(daughters.getFirst().getEncodedName());
214          parentNotCleaned.add(daughters.getSecond().getEncodedName());
215        }
216      }
217      return gcs;
218    } finally {
219      alreadyRunning.set(false);
220    }
221  }
222
223  /**
224   * Scan hbase:meta.
225   * @return Return generated {@link Report}
226   */
227  Report scanForReport() throws IOException {
228    ReportMakingVisitor visitor = new ReportMakingVisitor(this.services);
229    // Null tablename means scan all of meta.
230    MetaTableAccessor.scanMetaForTableRegions(this.services.getConnection(), visitor, null);
231    return visitor.getReport();
232  }
233
234  /**
235   * @return Returns last published Report that comes of last successful scan
236   *   of hbase:meta.
237   */
238  public Report getLastReport() {
239    return this.lastReport;
240  }
241
242  /**
243   * If merged region no longer holds reference to the merge regions, archive
244   * merge region on hdfs and perform deleting references in hbase:meta
245   * @return true if we delete references in merged region on hbase:meta and archive
246   *   the files on the file system
247   */
248  private boolean cleanMergeRegion(final RegionInfo mergedRegion, List<RegionInfo> parents)
249      throws IOException {
250    FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
251    Path rootdir = this.services.getMasterFileSystem().getRootDir();
252    Path tabledir = FSUtils.getTableDir(rootdir, mergedRegion.getTable());
253    TableDescriptor htd = getDescriptor(mergedRegion.getTable());
254    HRegionFileSystem regionFs = null;
255    try {
256      regionFs = HRegionFileSystem.openRegionFromFileSystem(
257          this.services.getConfiguration(), fs, tabledir, mergedRegion, true);
258    } catch (IOException e) {
259      LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());
260    }
261    if (regionFs == null || !regionFs.hasReferences(htd)) {
262      LOG.debug("Deleting parents ({}) from fs; merged child {} no longer holds references",
263           parents.stream().map(r -> RegionInfo.getShortNameToLog(r)).
264              collect(Collectors.joining(", ")),
265          mergedRegion);
266      ProcedureExecutor<MasterProcedureEnv> pe = this.services.getMasterProcedureExecutor();
267      pe.submitProcedure(new GCMultipleMergedRegionsProcedure(pe.getEnvironment(),
268          mergedRegion,  parents));
269      for (RegionInfo ri:  parents) {
270        // The above scheduled GCMultipleMergedRegionsProcedure does the below.
271        // Do we need this?
272        this.services.getAssignmentManager().getRegionStates().deleteRegion(ri);
273        this.services.getServerManager().removeRegion(ri);
274      }
275      return true;
276    }
277    return false;
278  }
279
280  /**
281   * Compare HRegionInfos in a way that has split parents sort BEFORE their daughters.
282   */
283  static class SplitParentFirstComparator implements Comparator<RegionInfo> {
284    Comparator<byte[]> rowEndKeyComparator = new Bytes.RowEndKeyComparator();
285    @Override
286    public int compare(RegionInfo left, RegionInfo right) {
287      // This comparator differs from the one RegionInfo in that it sorts
288      // parent before daughters.
289      if (left == null) {
290        return -1;
291      }
292      if (right == null) {
293        return 1;
294      }
295      // Same table name.
296      int result = left.getTable().compareTo(right.getTable());
297      if (result != 0) {
298        return result;
299      }
300      // Compare start keys.
301      result = Bytes.compareTo(left.getStartKey(), right.getStartKey());
302      if (result != 0) {
303        return result;
304      }
305      // Compare end keys, but flip the operands so parent comes first
306      result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey());
307
308      return result;
309    }
310  }
311
312  /**
313   * If daughters no longer hold reference to the parents, delete the parent.
314   * @param parent RegionInfo of split offlined parent
315   * @param rowContent Content of <code>parent</code> row in
316   * <code>metaRegionName</code>
317   * @return True if we removed <code>parent</code> from meta table and from
318   * the filesystem.
319   */
320  boolean cleanParent(final RegionInfo parent, Result rowContent)
321  throws IOException {
322    // Check whether it is a merged region and if it is clean of references.
323    if (MetaTableAccessor.hasMergeRegions(rowContent.rawCells())) {
324      // Wait until clean of merge parent regions first
325      return false;
326    }
327    // Run checks on each daughter split.
328    PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(rowContent);
329    Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst());
330    Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond());
331    if (hasNoReferences(a) && hasNoReferences(b)) {
332      String daughterA = daughters.getFirst() != null?
333          daughters.getFirst().getShortNameToLog(): "null";
334      String daughterB = daughters.getSecond() != null?
335          daughters.getSecond().getShortNameToLog(): "null";
336      LOG.debug("Deleting region " + parent.getShortNameToLog() +
337        " because daughters -- " + daughterA + ", " + daughterB +
338        " -- no longer hold references");
339      ProcedureExecutor<MasterProcedureEnv> pe = this.services.getMasterProcedureExecutor();
340      pe.submitProcedure(new GCRegionProcedure(pe.getEnvironment(), parent));
341      // Remove from in-memory states
342      this.services.getAssignmentManager().getRegionStates().deleteRegion(parent);
343      this.services.getServerManager().removeRegion(parent);
344      return true;
345    }
346    return false;
347  }
348
349  /**
350   * @param p A pair where the first boolean says whether or not the daughter
351   * region directory exists in the filesystem and then the second boolean says
352   * whether the daughter has references to the parent.
353   * @return True the passed <code>p</code> signifies no references.
354   */
355  private boolean hasNoReferences(final Pair<Boolean, Boolean> p) {
356    return !p.getFirst() || !p.getSecond();
357  }
358
359  /**
360   * Checks if a daughter region -- either splitA or splitB -- still holds
361   * references to parent.
362   * @param parent Parent region
363   * @param daughter Daughter region
364   * @return A pair where the first boolean says whether or not the daughter
365   *   region directory exists in the filesystem and then the second boolean says
366   *   whether the daughter has references to the parent.
367   */
368  private Pair<Boolean, Boolean> checkDaughterInFs(final RegionInfo parent,
369    final RegionInfo daughter)
370  throws IOException {
371    if (daughter == null)  {
372      return new Pair<>(Boolean.FALSE, Boolean.FALSE);
373    }
374
375    FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
376    Path rootdir = this.services.getMasterFileSystem().getRootDir();
377    Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable());
378
379    Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());
380
381    HRegionFileSystem regionFs;
382
383    try {
384      if (!FSUtils.isExists(fs, daughterRegionDir)) {
385        return new Pair<>(Boolean.FALSE, Boolean.FALSE);
386      }
387    } catch (IOException ioe) {
388      LOG.error("Error trying to determine if daughter region exists, " +
389               "assuming exists and has references", ioe);
390      return new Pair<>(Boolean.TRUE, Boolean.TRUE);
391    }
392
393    boolean references = false;
394    TableDescriptor parentDescriptor = getDescriptor(parent.getTable());
395    try {
396      regionFs = HRegionFileSystem.openRegionFromFileSystem(
397          this.services.getConfiguration(), fs, tabledir, daughter, true);
398
399      for (ColumnFamilyDescriptor family: parentDescriptor.getColumnFamilies()) {
400        if ((references = regionFs.hasReferences(family.getNameAsString()))) {
401          break;
402        }
403      }
404    } catch (IOException e) {
405      LOG.error("Error trying to determine referenced files from : " + daughter.getEncodedName()
406          + ", to: " + parent.getEncodedName() + " assuming has references", e);
407      return new Pair<>(Boolean.TRUE, Boolean.TRUE);
408    }
409    return new Pair<>(Boolean.TRUE, references);
410  }
411
412  private TableDescriptor getDescriptor(final TableName tableName) throws IOException {
413    return this.services.getTableDescriptors().get(tableName);
414  }
415
416  /**
417   * Checks if the specified region has merge qualifiers, if so, try to clean them.
418   * @return true if no info:merge* columns; i.e. the specified region doesn't have
419   *   any merge qualifiers.
420   */
421  public boolean cleanMergeQualifier(final RegionInfo region) throws IOException {
422    // Get merge regions if it is a merged region and already has merge qualifier
423    List<RegionInfo> parents = MetaTableAccessor.getMergeRegions(this.services.getConnection(),
424        region.getRegionName());
425    if (parents == null || parents.isEmpty()) {
426      // It doesn't have merge qualifier, no need to clean
427      return true;
428    }
429    return cleanMergeRegion(region, parents);
430  }
431
432  /**
433   * Report made by ReportMakingVisitor
434   */
435  public static class Report {
436    private final long now = EnvironmentEdgeManager.currentTime();
437
438    // Keep Map of found split parents. These are candidates for cleanup.
439    // Use a comparator that has split parents come before its daughters.
440    final Map<RegionInfo, Result> splitParents = new TreeMap<>(new SplitParentFirstComparator());
441    final Map<RegionInfo, Result> mergedRegions = new TreeMap<>(RegionInfo.COMPARATOR);
442    int count = 0;
443
444    private final List<Pair<RegionInfo, RegionInfo>> holes = new ArrayList<>();
445    private final List<Pair<RegionInfo, RegionInfo>> overlaps = new ArrayList<>();
446    private final List<Pair<RegionInfo, ServerName>> unknownServers = new ArrayList<>();
447    private final List<byte []> emptyRegionInfo = new ArrayList<>();
448
449    @VisibleForTesting
450    Report() {}
451
452    public long getCreateTime() {
453      return this.now;
454    }
455
456    public List<Pair<RegionInfo, RegionInfo>> getHoles() {
457      return this.holes;
458    }
459
460    public List<Pair<RegionInfo, RegionInfo>> getOverlaps() {
461      return this.overlaps;
462    }
463
464    public List<Pair<RegionInfo, ServerName>> getUnknownServers() {
465      return unknownServers;
466    }
467
468    public List<byte[]> getEmptyRegionInfo() {
469      return emptyRegionInfo;
470    }
471
472    /**
473     * @return True if an 'empty' lastReport -- no problems found.
474     */
475    public boolean isEmpty() {
476      return this.holes.isEmpty() && this.overlaps.isEmpty() && this.unknownServers.isEmpty() &&
477          this.emptyRegionInfo.isEmpty();
478    }
479
480    @Override
481    public String toString() {
482      StringBuffer sb = new StringBuffer();
483      for (Pair<RegionInfo, RegionInfo> p: this.holes) {
484        if (sb.length() > 0) {
485          sb.append(", ");
486        }
487        sb.append("hole=" + p.getFirst().getRegionNameAsString() + "/" +
488            p.getSecond().getRegionNameAsString());
489      }
490      for (Pair<RegionInfo, RegionInfo> p: this.overlaps) {
491        if (sb.length() > 0) {
492          sb.append(", ");
493        }
494        sb.append("overlap=" + p.getFirst().getRegionNameAsString() + "/" +
495            p.getSecond().getRegionNameAsString());
496      }
497      for (byte [] r: this.emptyRegionInfo) {
498        if (sb.length() > 0) {
499          sb.append(", ");
500        }
501        sb.append("empty=").append(Bytes.toStringBinary(r));
502      }
503      for (Pair<RegionInfo, ServerName> p: this.unknownServers) {
504        if (sb.length() > 0) {
505          sb.append(", ");
506        }
507        sb.append("unknown_server=").append(p.getSecond()).append("/").
508            append(p.getFirst().getRegionNameAsString());
509      }
510      return sb.toString();
511    }
512  }
513
514  /**
515   * Visitor we use in here in CatalogJanitor to go against hbase:meta table.
516   * Generates a Report made of a collection of split parents and counts of rows
517   * in the hbase:meta table. Also runs hbase:meta consistency checks to
518   * generate more report. Report is NOT ready until after this visitor has been
519   * {@link #close()}'d.
520   */
521  static class ReportMakingVisitor implements MetaTableAccessor.CloseableVisitor {
522    private final MasterServices services;
523    private volatile boolean closed;
524
525    /**
526     * Report is not done until after the close has been called.
527     * @see #close()
528     * @see #getReport()
529     */
530    private Report report = new Report();
531
532    /**
533     * RegionInfo from previous row.
534     */
535    private RegionInfo previous = null;
536
537    ReportMakingVisitor(MasterServices services) {
538      this.services = services;
539    }
540
541    /**
542     * Do not call until after {@link #close()}.
543     * Will throw a {@link RuntimeException} if you do.
544     */
545    Report getReport() {
546      if (!this.closed) {
547        throw new RuntimeException("Report not ready until after close()");
548      }
549      return this.report;
550    }
551
552    @Override
553    public boolean visit(Result r) {
554      if (r == null || r.isEmpty()) {
555        return true;
556      }
557      this.report.count++;
558      RegionInfo regionInfo = metaTableConsistencyCheck(r);
559      if (regionInfo != null) {
560        LOG.trace(regionInfo.toString());
561        if (regionInfo.isSplitParent()) { // splitParent means split and offline.
562          this.report.splitParents.put(regionInfo, r);
563        }
564        if (MetaTableAccessor.hasMergeRegions(r.rawCells())) {
565          this.report.mergedRegions.put(regionInfo, r);
566        }
567      }
568      // Returning true means "keep scanning"
569      return true;
570    }
571
572    /**
573     * Check row.
574     * @param metaTableRow Row from hbase:meta table.
575     * @return Returns default regioninfo found in row parse as a convenience to save
576     *   on having to do a double-parse of Result.
577     */
578    private RegionInfo metaTableConsistencyCheck(Result metaTableRow) {
579      RegionInfo ri;
580      // Locations comes back null if the RegionInfo field is empty.
581      // If locations is null, ensure the regioninfo is for sure empty before progressing.
582      // If really empty, report as missing regioninfo!  Otherwise, can run server check
583      // and get RegionInfo from locations.
584      RegionLocations locations = MetaTableAccessor.getRegionLocations(metaTableRow);
585      if (locations == null) {
586        ri = MetaTableAccessor.getRegionInfo(metaTableRow,
587            MetaTableAccessor.getRegionInfoColumn());
588      } else {
589        ri = locations.getDefaultRegionLocation().getRegion();
590        checkServer(metaTableRow.getRow(), locations);
591      }
592
593      if (ri == null) {
594        this.report.emptyRegionInfo.add(metaTableRow.getRow());
595        return ri;
596      }
597
598      if (!Bytes.equals(metaTableRow.getRow(), ri.getRegionName())) {
599        LOG.warn("INCONSISTENCY: Row name is not equal to serialized info:regioninfo content; " +
600                "row={} {}; See if RegionInfo is referenced in another hbase:meta row? Delete?",
601            Bytes.toStringBinary(metaTableRow.getRow()), ri.getRegionNameAsString());
602        return null;
603      }
604      // If table is disabled, skip integrity check.
605      if (!isTableDisabled(ri)) {
606        if (isTableTransition(ri)) {
607          // On table transition, look to see if last region was last in table
608          // and if this is the first. Report 'hole' if neither is true.
609          // HBCK1 used to have a special category for missing start or end keys.
610          // We'll just lump them in as 'holes'.
611          if ((this.previous != null && !this.previous.isLast()) || !ri.isFirst()) {
612            addHole(this.previous == null? RegionInfo.UNDEFINED: this.previous, ri);
613          }
614        } else {
615          if (!this.previous.isNext(ri)) {
616            if (this.previous.isOverlap(ri)) {
617              addOverlap(this.previous, ri);
618            } else {
619              addHole(this.previous, ri);
620            }
621          }
622        }
623      }
624      this.previous = ri;
625      return ri;
626    }
627
628    private void addOverlap(RegionInfo a, RegionInfo b) {
629      this.report.overlaps.add(new Pair<>(a, b));
630    }
631
632    private void addHole(RegionInfo a, RegionInfo b) {
633      this.report.holes.add(new Pair<>(a, b));
634    }
635
636    /**
637     * @return True if table is disabled or disabling; defaults false!
638     */
639    boolean isTableDisabled(RegionInfo ri) {
640      if (ri == null) {
641        return false;
642      }
643      if (this.services == null) {
644        return false;
645      }
646      if (this.services.getTableStateManager() == null) {
647        return false;
648      }
649      TableState state = null;
650      try {
651        state = this.services.getTableStateManager().getTableState(ri.getTable());
652      } catch (IOException e) {
653        LOG.warn("Failed getting table state", e);
654      }
655      return state != null && state.isDisabledOrDisabling();
656    }
657
658    /**
659     * Run through referenced servers and save off unknown and the dead.
660     */
661    private void checkServer(byte [] metaTableRow, RegionLocations locations) {
662      if (this.services == null) {
663        // Can't do this test if no services.
664        return;
665      }
666      if (locations == null) {
667        return;
668      }
669      // Check referenced servers are known/online.
670      for (HRegionLocation location: locations.getRegionLocations()) {
671        ServerName sn = location.getServerName();
672        if (sn == null) {
673          continue;
674        }
675        // skip the offline regions which belong to disabled table.
676        if (isTableDisabled(location.getRegion())) {
677          continue;
678        }
679        ServerManager.ServerLiveState state = this.services.getServerManager().
680            isServerKnownAndOnline(sn);
681        switch (state) {
682          case UNKNOWN:
683            this.report.unknownServers.add(new Pair(location.getRegion(), sn));
684            break;
685
686          default:
687            break;
688        }
689      }
690    }
691
692    /**
693     * @return True iff first row in hbase:meta or if we've broached a new table in hbase:meta
694     */
695    private boolean isTableTransition(RegionInfo ri) {
696      return this.previous == null ||
697          !this.previous.getTable().equals(ri.getTable());
698    }
699
700    @Override
701    public void close() throws IOException {
702      // This is a table transition... after the last region. Check previous.
703      // Should be last region. If not, its a hole on end of laster table.
704      if (this.previous != null && !this.previous.isLast()) {
705        addHole(this.previous, RegionInfo.UNDEFINED);
706      }
707      this.closed = true;
708    }
709  }
710
711  private static void checkLog4jProperties() {
712    String filename = "log4j.properties";
713    try {
714      final InputStream inStream =
715          CatalogJanitor.class.getClassLoader().getResourceAsStream(filename);
716      if (inStream != null) {
717        new Properties().load(inStream);
718      } else {
719        System.out.println("No " + filename + " on classpath; Add one else no logging output!");
720      }
721    } catch (IOException e) {
722      LOG.error("Log4j check failed", e);
723    }
724  }
725
726  /**
727   * For testing against a cluster.
728   * Doesn't have a MasterServices context so does not report on good vs bad servers.
729   */
730  public static void main(String [] args) throws IOException {
731    checkLog4jProperties();
732    ReportMakingVisitor visitor = new ReportMakingVisitor(null);
733    Configuration configuration = HBaseConfiguration.create();
734    configuration.setBoolean("hbase.defaults.for.version.skip", true);
735    try (Connection connection = ConnectionFactory.createConnection(configuration)) {
736      /* Used to generate an overlap.
737      */
738      Get g = new Get(Bytes.toBytes("t2,40,1564119846424.1db8c57d64e0733e0f027aaeae7a0bf0."));
739      g.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
740      try (Table t = connection.getTable(TableName.META_TABLE_NAME)) {
741        Result r = t.get(g);
742        byte [] row = g.getRow();
743        row[row.length - 2] <<= row[row.length - 2];
744        Put p = new Put(g.getRow());
745        p.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
746            r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
747        t.put(p);
748      }
749      MetaTableAccessor.scanMetaForTableRegions(connection, visitor, null);
750      Report report = visitor.getReport();
751      LOG.info(report != null? report.toString(): "empty");
752    }
753  }
754}