001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.janitor;
019
020import java.io.IOException;
021import java.io.InputStream;
022import java.util.Comparator;
023import java.util.HashSet;
024import java.util.List;
025import java.util.Map;
026import java.util.Properties;
027import java.util.concurrent.atomic.AtomicBoolean;
028import java.util.stream.Collectors;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.HBaseConfiguration;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.MetaTableAccessor;
035import org.apache.hadoop.hbase.ScheduledChore;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.Connection;
038import org.apache.hadoop.hbase.client.ConnectionFactory;
039import org.apache.hadoop.hbase.client.Get;
040import org.apache.hadoop.hbase.client.Put;
041import org.apache.hadoop.hbase.client.RegionInfo;
042import org.apache.hadoop.hbase.client.Result;
043import org.apache.hadoop.hbase.client.Table;
044import org.apache.hadoop.hbase.client.TableDescriptor;
045import org.apache.hadoop.hbase.master.MasterServices;
046import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
047import org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure;
048import org.apache.hadoop.hbase.master.assignment.GCRegionProcedure;
049import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
050import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
051import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
052import org.apache.hadoop.hbase.util.Bytes;
053import org.apache.hadoop.hbase.util.CommonFSUtils;
054import org.apache.hadoop.hbase.util.Pair;
055import org.apache.hadoop.hbase.util.PairOfSameType;
056import org.apache.hadoop.hbase.util.Threads;
057import org.apache.yetus.audience.InterfaceAudience;
058import org.slf4j.Logger;
059import org.slf4j.LoggerFactory;
060
061/**
062 * A janitor for the catalog tables. Scans the <code>hbase:meta</code> catalog table on a period.
063 * Makes a lastReport on state of hbase:meta. Looks for unused regions to garbage collect. Scan of
064 * hbase:meta runs if we are NOT in maintenance mode, if we are NOT shutting down, AND if the
065 * assignmentmanager is loaded. Playing it safe, we will garbage collect no-longer needed region
066 * references only if there are no regions-in-transition (RIT).
067 */
068// TODO: Only works with single hbase:meta region currently. Fix.
069// TODO: Should it start over every time? Could it continue if runs into problem? Only if
070// problem does not mess up 'results'.
071// TODO: Do more by way of 'repair'; see note on unknownServers below.
072@InterfaceAudience.Private
073public class CatalogJanitor extends ScheduledChore {
074
075  public static final int DEFAULT_HBASE_CATALOGJANITOR_INTERVAL = 300 * 1000;
076
077  private static final Logger LOG = LoggerFactory.getLogger(CatalogJanitor.class.getName());
078
079  private final AtomicBoolean alreadyRunning = new AtomicBoolean(false);
080  private final AtomicBoolean enabled = new AtomicBoolean(true);
081  private final MasterServices services;
082
083  /**
084   * Saved report from last hbase:meta scan to completion. May be stale if having trouble completing
085   * scan. Check its date.
086   */
087  private volatile CatalogJanitorReport lastReport;
088
089  public CatalogJanitor(final MasterServices services) {
090    super("CatalogJanitor-" + services.getServerName().toShortString(), services,
091      services.getConfiguration().getInt("hbase.catalogjanitor.interval",
092        DEFAULT_HBASE_CATALOGJANITOR_INTERVAL));
093    this.services = services;
094  }
095
096  @Override
097  protected boolean initialChore() {
098    try {
099      if (getEnabled()) {
100        scan();
101      }
102    } catch (IOException e) {
103      LOG.warn("Failed initial janitorial scan of hbase:meta table", e);
104      return false;
105    }
106    return true;
107  }
108
109  public boolean setEnabled(final boolean enabled) {
110    boolean alreadyEnabled = this.enabled.getAndSet(enabled);
111    // If disabling is requested on an already enabled chore, we could have an active
112    // scan still going on, callers might not be aware of that and do further action thinkng
113    // that no action would be from this chore. In this case, the right action is to wait for
114    // the active scan to complete before exiting this function.
115    if (!enabled && alreadyEnabled) {
116      while (alreadyRunning.get()) {
117        Threads.sleepWithoutInterrupt(100);
118      }
119    }
120    return alreadyEnabled;
121  }
122
123  public boolean getEnabled() {
124    return this.enabled.get();
125  }
126
127  @Override
128  protected void chore() {
129    try {
130      AssignmentManager am = this.services.getAssignmentManager();
131      if (
132        getEnabled() && !this.services.isInMaintenanceMode()
133          && !this.services.getServerManager().isClusterShutdown() && isMetaLoaded(am)
134      ) {
135        scan();
136      } else {
137        LOG.warn("CatalogJanitor is disabled! Enabled=" + getEnabled() + ", maintenanceMode="
138          + this.services.isInMaintenanceMode() + ", am=" + am + ", metaLoaded=" + isMetaLoaded(am)
139          + ", hasRIT=" + isRIT(am) + " clusterShutDown="
140          + this.services.getServerManager().isClusterShutdown());
141      }
142    } catch (IOException e) {
143      LOG.warn("Failed janitorial scan of hbase:meta table", e);
144    }
145  }
146
147  private static boolean isMetaLoaded(AssignmentManager am) {
148    return am != null && am.isMetaLoaded();
149  }
150
151  private static boolean isRIT(AssignmentManager am) {
152    return isMetaLoaded(am) && am.hasRegionsInTransition();
153  }
154
155  /**
156   * Run janitorial scan of catalog <code>hbase:meta</code> table looking for garbage to collect.
157   * @return How many items gc'd whether for merge or split. Returns -1 if previous scan is in
158   *         progress.
159   */
160  public int scan() throws IOException {
161    int gcs = 0;
162    try {
163      if (!alreadyRunning.compareAndSet(false, true)) {
164        if (LOG.isDebugEnabled()) {
165          LOG.debug("CatalogJanitor already running");
166        }
167        // -1 indicates previous scan is in progress
168        return -1;
169      }
170      this.lastReport = scanForReport();
171      if (!this.lastReport.isEmpty()) {
172        LOG.warn(this.lastReport.toString());
173      } else {
174        if (LOG.isDebugEnabled()) {
175          LOG.debug(this.lastReport.toString());
176        }
177      }
178
179      updateAssignmentManagerMetrics();
180
181      Map<RegionInfo, Result> mergedRegions = this.lastReport.mergedRegions;
182      for (Map.Entry<RegionInfo, Result> e : mergedRegions.entrySet()) {
183        if (this.services.isInMaintenanceMode()) {
184          // Stop cleaning if the master is in maintenance mode
185          LOG.debug("In maintenance mode, not cleaning");
186          break;
187        }
188
189        List<RegionInfo> parents = MetaTableAccessor.getMergeRegions(e.getValue().rawCells());
190        if (parents != null && cleanMergeRegion(this.services, e.getKey(), parents)) {
191          gcs++;
192        }
193      }
194      // Clean split parents
195      Map<RegionInfo, Result> splitParents = this.lastReport.splitParents;
196
197      // Now work on our list of found parents. See if any we can clean up.
198      HashSet<String> parentNotCleaned = new HashSet<>();
199      for (Map.Entry<RegionInfo, Result> e : splitParents.entrySet()) {
200        if (this.services.isInMaintenanceMode()) {
201          // Stop cleaning if the master is in maintenance mode
202          if (LOG.isDebugEnabled()) {
203            LOG.debug("In maintenance mode, not cleaning");
204          }
205          break;
206        }
207
208        if (
209          !parentNotCleaned.contains(e.getKey().getEncodedName())
210            && cleanParent(e.getKey(), e.getValue())
211        ) {
212          gcs++;
213        } else {
214          // We could not clean the parent, so it's daughters should not be
215          // cleaned either (HBASE-6160)
216          PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(e.getValue());
217          parentNotCleaned.add(daughters.getFirst().getEncodedName());
218          parentNotCleaned.add(daughters.getSecond().getEncodedName());
219        }
220      }
221      return gcs;
222    } finally {
223      alreadyRunning.set(false);
224    }
225  }
226
227  /**
228   * Scan hbase:meta.
229   * @return Return generated {@link CatalogJanitorReport}
230   */
231  // will be override in tests.
232  protected CatalogJanitorReport scanForReport() throws IOException {
233    ReportMakingVisitor visitor = new ReportMakingVisitor(this.services);
234    // Null tablename means scan all of meta.
235    MetaTableAccessor.scanMetaForTableRegions(this.services.getConnection(), visitor, null);
236    return visitor.getReport();
237  }
238
239  /** Returns Returns last published Report that comes of last successful scan of hbase:meta. */
240  public CatalogJanitorReport getLastReport() {
241    return this.lastReport;
242  }
243
244  /**
245   * If merged region no longer holds reference to the merge regions, archive merge region on hdfs
246   * and perform deleting references in hbase:meta
247   * @return true if we delete references in merged region on hbase:meta and archive the files on
248   *         the file system
249   */
250  static boolean cleanMergeRegion(MasterServices services, final RegionInfo mergedRegion,
251    List<RegionInfo> parents) throws IOException {
252    if (LOG.isDebugEnabled()) {
253      LOG.debug("Cleaning merged region {}", mergedRegion);
254    }
255
256    Pair<Boolean, Boolean> result =
257      checkRegionReferences(services, mergedRegion.getTable(), mergedRegion);
258
259    if (hasNoReferences(result)) {
260      if (LOG.isDebugEnabled()) {
261        LOG.debug(
262          "Deleting parents ({}) from fs; merged child {} no longer holds references", parents
263            .stream().map(r -> RegionInfo.getShortNameToLog(r)).collect(Collectors.joining(", ")),
264          mergedRegion);
265      }
266
267      ProcedureExecutor<MasterProcedureEnv> pe = services.getMasterProcedureExecutor();
268      GCMultipleMergedRegionsProcedure mergeRegionProcedure =
269        new GCMultipleMergedRegionsProcedure(pe.getEnvironment(), mergedRegion, parents);
270      pe.submitProcedure(mergeRegionProcedure);
271      if (LOG.isDebugEnabled()) {
272        LOG.debug("Submitted procedure {} for merged region {}", mergeRegionProcedure,
273          mergedRegion);
274      }
275      return true;
276    } else {
277      if (LOG.isDebugEnabled()) {
278        LOG.debug(
279          "Deferring cleanup up of {} parents of merged region {}, because references "
280            + "still exist in merged region or we encountered an exception in checking",
281          parents.size(), mergedRegion.getEncodedName());
282      }
283    }
284
285    return false;
286  }
287
288  /**
289   * Compare HRegionInfos in a way that has split parents sort BEFORE their daughters.
290   */
291  static class SplitParentFirstComparator implements Comparator<RegionInfo> {
292    Comparator<byte[]> rowEndKeyComparator = new Bytes.RowEndKeyComparator();
293
294    @Override
295    public int compare(RegionInfo left, RegionInfo right) {
296      // This comparator differs from the one RegionInfo in that it sorts
297      // parent before daughters.
298      if (left == null) {
299        return -1;
300      }
301      if (right == null) {
302        return 1;
303      }
304      // Same table name.
305      int result = left.getTable().compareTo(right.getTable());
306      if (result != 0) {
307        return result;
308      }
309      // Compare start keys.
310      result = Bytes.compareTo(left.getStartKey(), right.getStartKey());
311      if (result != 0) {
312        return result;
313      }
314      // Compare end keys, but flip the operands so parent comes first
315      result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey());
316
317      return result;
318    }
319  }
320
321  static boolean cleanParent(MasterServices services, RegionInfo parent, Result rowContent)
322    throws IOException {
323    if (LOG.isDebugEnabled()) {
324      LOG.debug("Cleaning parent region {}", parent);
325    }
326    // Check whether it is a merged region and if it is clean of references.
327    if (MetaTableAccessor.hasMergeRegions(rowContent.rawCells())) {
328      // Wait until clean of merge parent regions first
329      if (LOG.isDebugEnabled()) {
330        LOG.debug("Region {} has merge parents, cleaning them first", parent);
331      }
332      return false;
333    }
334    // Run checks on each daughter split.
335    PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(rowContent);
336    Pair<Boolean, Boolean> a =
337      checkRegionReferences(services, parent.getTable(), daughters.getFirst());
338    Pair<Boolean, Boolean> b =
339      checkRegionReferences(services, parent.getTable(), daughters.getSecond());
340    if (hasNoReferences(a) && hasNoReferences(b)) {
341      String daughterA =
342        daughters.getFirst() != null ? daughters.getFirst().getShortNameToLog() : "null";
343      String daughterB =
344        daughters.getSecond() != null ? daughters.getSecond().getShortNameToLog() : "null";
345      if (LOG.isDebugEnabled()) {
346        LOG.debug("Deleting region " + parent.getShortNameToLog() + " because daughters -- "
347          + daughterA + ", " + daughterB + " -- no longer hold references");
348      }
349      ProcedureExecutor<MasterProcedureEnv> pe = services.getMasterProcedureExecutor();
350      GCRegionProcedure gcRegionProcedure = new GCRegionProcedure(pe.getEnvironment(), parent);
351      pe.submitProcedure(gcRegionProcedure);
352      if (LOG.isDebugEnabled()) {
353        LOG.debug("Submitted procedure {} for split parent {}", gcRegionProcedure, parent);
354      }
355      return true;
356    } else {
357      if (LOG.isDebugEnabled()) {
358        if (!hasNoReferences(a)) {
359          LOG.debug("Deferring removal of region {} because daughter {} still has references",
360            parent, daughters.getFirst());
361        }
362        if (!hasNoReferences(b)) {
363          LOG.debug("Deferring removal of region {} because daughter {} still has references",
364            parent, daughters.getSecond());
365        }
366      }
367    }
368    return false;
369  }
370
371  /**
372   * If daughters no longer hold reference to the parents, delete the parent.
373   * @param parent     RegionInfo of split offlined parent
374   * @param rowContent Content of <code>parent</code> row in <code>metaRegionName</code>
375   * @return True if we removed <code>parent</code> from meta table and from the filesystem.
376   */
377  private boolean cleanParent(final RegionInfo parent, Result rowContent) throws IOException {
378    return cleanParent(services, parent, rowContent);
379  }
380
381  /**
382   * @param p A pair where the first boolean says whether or not the daughter region directory
383   *          exists in the filesystem and then the second boolean says whether the daughter has
384   *          references to the parent.
385   * @return True the passed <code>p</code> signifies no references.
386   */
387  private static boolean hasNoReferences(final Pair<Boolean, Boolean> p) {
388    return !p.getFirst() || !p.getSecond();
389  }
390
391  /**
392   * Checks if a region still holds references to parent.
393   * @param tableName The table for the region
394   * @param region    The region to check
395   * @return A pair where the first boolean says whether the region directory exists in the
396   *         filesystem and then the second boolean says whether the region has references to a
397   *         parent.
398   */
399  private static Pair<Boolean, Boolean> checkRegionReferences(MasterServices services,
400    TableName tableName, RegionInfo region) throws IOException {
401    if (region == null) {
402      return new Pair<>(Boolean.FALSE, Boolean.FALSE);
403    }
404
405    FileSystem fs = services.getMasterFileSystem().getFileSystem();
406    Path rootdir = services.getMasterFileSystem().getRootDir();
407    Path tabledir = CommonFSUtils.getTableDir(rootdir, tableName);
408    Path regionDir = new Path(tabledir, region.getEncodedName());
409
410    try {
411      if (!CommonFSUtils.isExists(fs, regionDir)) {
412        return new Pair<>(Boolean.FALSE, Boolean.FALSE);
413      }
414    } catch (IOException ioe) {
415      LOG.error("Error trying to determine if region exists, assuming exists and has references",
416        ioe);
417      return new Pair<>(Boolean.TRUE, Boolean.TRUE);
418    }
419
420    TableDescriptor tableDescriptor = services.getTableDescriptors().get(tableName);
421    try {
422      HRegionFileSystem regionFs = HRegionFileSystem
423        .openRegionFromFileSystem(services.getConfiguration(), fs, tabledir, region, true);
424      boolean references = regionFs.hasReferences(tableDescriptor);
425      return new Pair<>(Boolean.TRUE, references);
426    } catch (IOException e) {
427      LOG.error("Error trying to determine if region {} has references, assuming it does",
428        region.getEncodedName(), e);
429      return new Pair<>(Boolean.TRUE, Boolean.TRUE);
430    }
431  }
432
433  private void updateAssignmentManagerMetrics() {
434    services.getAssignmentManager().getAssignmentManagerMetrics()
435      .updateHoles(lastReport.getHoles().size());
436    services.getAssignmentManager().getAssignmentManagerMetrics()
437      .updateOverlaps(lastReport.getOverlaps().size());
438    services.getAssignmentManager().getAssignmentManagerMetrics()
439      .updateUnknownServerRegions(lastReport.getUnknownServers().size());
440    services.getAssignmentManager().getAssignmentManagerMetrics()
441      .updateEmptyRegionInfoRegions(lastReport.getEmptyRegionInfo().size());
442  }
443
444  private static void checkLog4jProperties() {
445    String filename = "log4j.properties";
446    try (final InputStream inStream =
447      CatalogJanitor.class.getClassLoader().getResourceAsStream(filename)) {
448      if (inStream != null) {
449        new Properties().load(inStream);
450      } else {
451        System.out.println("No " + filename + " on classpath; Add one else no logging output!");
452      }
453    } catch (IOException e) {
454      LOG.error("Log4j check failed", e);
455    }
456  }
457
458  /**
459   * For testing against a cluster. Doesn't have a MasterServices context so does not report on good
460   * vs bad servers.
461   */
462  public static void main(String[] args) throws IOException {
463    checkLog4jProperties();
464    ReportMakingVisitor visitor = new ReportMakingVisitor(null);
465    Configuration configuration = HBaseConfiguration.create();
466    configuration.setBoolean("hbase.defaults.for.version.skip", true);
467    try (Connection connection = ConnectionFactory.createConnection(configuration)) {
468      /*
469       * Used to generate an overlap.
470       */
471      Get g = new Get(Bytes.toBytes("t2,40,1564119846424.1db8c57d64e0733e0f027aaeae7a0bf0."));
472      g.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
473      try (Table t = connection.getTable(TableName.META_TABLE_NAME)) {
474        Result r = t.get(g);
475        byte[] row = g.getRow();
476        row[row.length - 2] <<= row[row.length - 2];
477        Put p = new Put(g.getRow());
478        p.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
479          r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
480        t.put(p);
481      }
482      MetaTableAccessor.scanMetaForTableRegions(connection, visitor, null);
483      CatalogJanitorReport report = visitor.getReport();
484      LOG.info(report != null ? report.toString() : "empty");
485    }
486  }
487}