001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import java.io.Closeable;
021import java.io.FileNotFoundException;
022import java.io.IOException;
023import java.io.InterruptedIOException;
024import java.io.PrintWriter;
025import java.io.StringWriter;
026import java.net.InetAddress;
027import java.net.URI;
028import java.util.ArrayList;
029import java.util.Collection;
030import java.util.Collections;
031import java.util.Comparator;
032import java.util.EnumSet;
033import java.util.HashMap;
034import java.util.HashSet;
035import java.util.Iterator;
036import java.util.List;
037import java.util.Locale;
038import java.util.Map;
039import java.util.Map.Entry;
040import java.util.Objects;
041import java.util.Optional;
042import java.util.Set;
043import java.util.SortedMap;
044import java.util.TreeMap;
045import java.util.Vector;
046import java.util.concurrent.Callable;
047import java.util.concurrent.ConcurrentSkipListMap;
048import java.util.concurrent.ExecutionException;
049import java.util.concurrent.ExecutorService;
050import java.util.concurrent.Executors;
051import java.util.concurrent.Future;
052import java.util.concurrent.FutureTask;
053import java.util.concurrent.ScheduledThreadPoolExecutor;
054import java.util.concurrent.TimeUnit;
055import java.util.concurrent.TimeoutException;
056import java.util.concurrent.atomic.AtomicBoolean;
057import java.util.concurrent.atomic.AtomicInteger;
058import java.util.stream.Collectors;
059import org.apache.commons.io.IOUtils;
060import org.apache.commons.lang3.StringUtils;
061import org.apache.hadoop.conf.Configuration;
062import org.apache.hadoop.conf.Configured;
063import org.apache.hadoop.fs.FSDataOutputStream;
064import org.apache.hadoop.fs.FileStatus;
065import org.apache.hadoop.fs.FileSystem;
066import org.apache.hadoop.fs.Path;
067import org.apache.hadoop.fs.permission.FsAction;
068import org.apache.hadoop.fs.permission.FsPermission;
069import org.apache.hadoop.hbase.Abortable;
070import org.apache.hadoop.hbase.CatalogFamilyFormat;
071import org.apache.hadoop.hbase.Cell;
072import org.apache.hadoop.hbase.CellUtil;
073import org.apache.hadoop.hbase.ClientMetaTableAccessor;
074import org.apache.hadoop.hbase.ClusterMetrics;
075import org.apache.hadoop.hbase.ClusterMetrics.Option;
076import org.apache.hadoop.hbase.HBaseConfiguration;
077import org.apache.hadoop.hbase.HBaseInterfaceAudience;
078import org.apache.hadoop.hbase.HConstants;
079import org.apache.hadoop.hbase.HRegionLocation;
080import org.apache.hadoop.hbase.KeyValue;
081import org.apache.hadoop.hbase.MasterNotRunningException;
082import org.apache.hadoop.hbase.MetaTableAccessor;
083import org.apache.hadoop.hbase.RegionLocations;
084import org.apache.hadoop.hbase.ServerName;
085import org.apache.hadoop.hbase.TableName;
086import org.apache.hadoop.hbase.TableNotFoundException;
087import org.apache.hadoop.hbase.ZooKeeperConnectionException;
088import org.apache.hadoop.hbase.client.Admin;
089import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
090import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
091import org.apache.hadoop.hbase.client.Connection;
092import org.apache.hadoop.hbase.client.ConnectionFactory;
093import org.apache.hadoop.hbase.client.Delete;
094import org.apache.hadoop.hbase.client.Get;
095import org.apache.hadoop.hbase.client.Put;
096import org.apache.hadoop.hbase.client.RegionInfo;
097import org.apache.hadoop.hbase.client.RegionInfoBuilder;
098import org.apache.hadoop.hbase.client.RegionLocator;
099import org.apache.hadoop.hbase.client.RegionReplicaUtil;
100import org.apache.hadoop.hbase.client.Result;
101import org.apache.hadoop.hbase.client.ResultScanner;
102import org.apache.hadoop.hbase.client.RowMutations;
103import org.apache.hadoop.hbase.client.Scan;
104import org.apache.hadoop.hbase.client.Table;
105import org.apache.hadoop.hbase.client.TableDescriptor;
106import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
107import org.apache.hadoop.hbase.client.TableState;
108import org.apache.hadoop.hbase.io.FileLink;
109import org.apache.hadoop.hbase.io.HFileLink;
110import org.apache.hadoop.hbase.io.hfile.CacheConfig;
111import org.apache.hadoop.hbase.io.hfile.HFile;
112import org.apache.hadoop.hbase.master.RegionState;
113import org.apache.hadoop.hbase.regionserver.HRegion;
114import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
115import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
116import org.apache.hadoop.hbase.replication.ReplicationException;
117import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
118import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
119import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
120import org.apache.hadoop.hbase.security.UserProvider;
121import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
122import org.apache.hadoop.hbase.util.HbckErrorReporter.ERROR_CODE;
123import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
124import org.apache.hadoop.hbase.util.hbck.ReplicationChecker;
125import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
126import org.apache.hadoop.hbase.wal.WALSplitUtil;
127import org.apache.hadoop.hbase.zookeeper.ZKUtil;
128import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
129import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
130import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
131import org.apache.hadoop.ipc.RemoteException;
132import org.apache.hadoop.security.AccessControlException;
133import org.apache.hadoop.security.UserGroupInformation;
134import org.apache.hadoop.util.ReflectionUtils;
135import org.apache.hadoop.util.Tool;
136import org.apache.hadoop.util.ToolRunner;
137import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
138import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
139import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
140import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
141import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
142import org.apache.yetus.audience.InterfaceAudience;
143import org.apache.yetus.audience.InterfaceStability;
144import org.apache.zookeeper.KeeperException;
145import org.slf4j.Logger;
146import org.slf4j.LoggerFactory;
147
148/**
149 * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
150 * table integrity problems in a corrupted HBase. This tool was written for hbase-1.x. It does not
151 * work with hbase-2.x; it can read state but is not allowed to change state; i.e. effect 'repair'.
152 * Even though it can 'read' state, given how so much has changed in how hbase1 and hbase2 operate,
153 * it will often misread. See hbck2 (HBASE-19121) for a hbck tool for hbase2. This class is
154 * deprecated.
155 *
156 * <p>
157 * Region consistency checks verify that hbase:meta, region deployment on region
158 * servers and the state of data in HDFS (.regioninfo files) all are in
159 * accordance.
160 * <p>
161 * Table integrity checks verify that all possible row keys resolve to exactly
162 * one region of a table.  This means there are no individual degenerate
163 * or backwards regions; no holes between regions; and that there are no
164 * overlapping regions.
165 * <p>
166 * The general repair strategy works in two phases:
167 * <ol>
168 * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
169 * <li> Repair Region Consistency with hbase:meta and assignments
170 * </ol>
171 * <p>
172 * For table integrity repairs, the tables' region directories are scanned
173 * for .regioninfo files.  Each table's integrity is then verified.  If there
174 * are any orphan regions (regions with no .regioninfo files) or holes, new
175 * regions are fabricated.  Backwards regions are sidelined as well as empty
176 * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
177 * a new region is created and all data is merged into the new region.
178 * <p>
179 * Table integrity repairs deal solely with HDFS and could potentially be done
180 * offline -- the hbase region servers or master do not need to be running.
181 * This phase can eventually be used to completely reconstruct the hbase:meta table in
182 * an offline fashion.
183 * <p>
184 * Region consistency requires three conditions -- 1) valid .regioninfo file
185 * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
186 * and 3) a region is deployed only at the regionserver that was assigned to
187 * with proper state in the master.
188 * <p>
189 * Region consistency repairs require hbase to be online so that hbck can
190 * contact the HBase master and region servers.  The hbck#connect() method must
191 * first be called successfully.  Much of the region consistency information
192 * is transient and less risky to repair.
193 * <p>
194 * If hbck is run from the command line, there are a handful of arguments that
195 * can be used to limit the kinds of repairs hbck will do.  See the code in
196 * {@link #printUsageAndExit()} for more details.
197 * @deprecated For removal in hbase-4.0.0. Use HBCK2 instead.
198 */
199@Deprecated
200@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
201@InterfaceStability.Evolving
202public class HBaseFsck extends Configured implements Closeable {
203  public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
204  public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
205  private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
206  private static boolean rsSupportsOffline = true;
207  private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
208  private static final int DEFAULT_MAX_MERGE = 5;
209
210  /**
211   * Here is where hbase-1.x used to default the lock for hbck1.
212   * It puts in place a lock when it goes to write/make changes.
213   */
214  @InterfaceAudience.Private
215  public static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
216  private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5;
217  private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds
218  private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds
219  // We have to set the timeout value > HdfsConstants.LEASE_SOFTLIMIT_PERIOD.
220  // In HADOOP-2.6 and later, the Namenode proxy now created with custom RetryPolicy for
221  // AlreadyBeingCreatedException which is implies timeout on this operations up to
222  // HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds).
223  private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds
224  private static final int DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS = 5;
225  private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds
226  private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds
227
228  /**********************
229   * Internal resources
230   **********************/
231  private static final Logger LOG = LoggerFactory.getLogger(HBaseFsck.class.getName());
232  private ClusterMetrics status;
233  private Connection connection;
234  private Admin admin;
235  private Table meta;
236  // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
237  protected ExecutorService executor;
238  private long startMillis = EnvironmentEdgeManager.currentTime();
239  private HFileCorruptionChecker hfcc;
240  private int retcode = 0;
241  private Path HBCK_LOCK_PATH;
242  private FSDataOutputStream hbckOutFd;
243  // This lock is to prevent cleanup of balancer resources twice between
244  // ShutdownHook and the main code. We cleanup only if the connect() is
245  // successful
246  private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
247
248  // Unsupported options in HBase 2.0+
249  private static final Set<String> unsupportedOptionsInV2 = Sets.newHashSet("-fix",
250      "-fixAssignments", "-fixMeta", "-fixHdfsHoles", "-fixHdfsOrphans", "-fixTableOrphans",
251      "-fixHdfsOverlaps", "-sidelineBigOverlaps", "-fixSplitParents", "-removeParents",
252      "-fixEmptyMetaCells", "-repair", "-repairHoles", "-maxOverlapsToSideline", "-maxMerge");
253
254  /***********
255   * Options
256   ***********/
257  private static boolean details = false; // do we display the full report
258  private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
259  private static boolean forceExclusive = false; // only this hbck can modify HBase
260  private boolean fixAssignments = false; // fix assignment errors?
261  private boolean fixMeta = false; // fix meta errors?
262  private boolean checkHdfs = true; // load and check fs consistency?
263  private boolean fixHdfsHoles = false; // fix fs holes?
264  private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
265  private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
266  private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
267  private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
268  private boolean fixSplitParents = false; // fix lingering split parents
269  private boolean removeParents = false; // remove split parents
270  private boolean fixReferenceFiles = false; // fix lingering reference store file
271  private boolean fixHFileLinks = false; // fix lingering HFileLinks
272  private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
273  private boolean fixReplication = false; // fix undeleted replication queues for removed peer
274  private boolean cleanReplicationBarrier = false; // clean replication barriers of a table
275  private boolean fixAny = false; // Set to true if any of the fix is required.
276
277  // limit checking/fixes to listed tables, if empty attempt to check/fix all
278  // hbase:meta are always checked
279  private Set<TableName> tablesIncluded = new HashSet<>();
280  private TableName cleanReplicationBarrierTable;
281  private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
282  // maximum number of overlapping regions to sideline
283  private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE;
284  private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
285  private Path sidelineDir = null;
286
287  private boolean rerun = false; // if we tried to fix something, rerun hbck
288  private static boolean summary = false; // if we want to print less output
289  private boolean checkMetaOnly = false;
290  private boolean checkRegionBoundaries = false;
291  private boolean ignorePreCheckPermission = false; // if pre-check permission
292
293  /*********
294   * State
295   *********/
296  final private HbckErrorReporter errors;
297  int fixes = 0;
298
299  /**
300   * This map contains the state of all hbck items.  It maps from encoded region
301   * name to HbckRegionInfo structure.  The information contained in HbckRegionInfo is used
302   * to detect and correct consistency (hdfs/meta/deployment) problems.
303   */
304  private TreeMap<String, HbckRegionInfo> regionInfoMap = new TreeMap<>();
305  // Empty regioninfo qualifiers in hbase:meta
306  private Set<Result> emptyRegionInfoQualifiers = new HashSet<>();
307
308  /**
309   * This map from Tablename -> TableInfo contains the structures necessary to
310   * detect table consistency problems (holes, dupes, overlaps).  It is sorted
311   * to prevent dupes.
312   *
313   * If tablesIncluded is empty, this map contains all tables.
314   * Otherwise, it contains only meta tables and tables in tablesIncluded,
315   * unless checkMetaOnly is specified, in which case, it contains only
316   * the meta table
317   */
318  private SortedMap<TableName, HbckTableInfo> tablesInfo = new ConcurrentSkipListMap<>();
319
320  /**
321   * When initially looking at HDFS, we attempt to find any orphaned data.
322   */
323  private List<HbckRegionInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<>());
324
325  private Map<TableName, Set<String>> orphanTableDirs = new HashMap<>();
326  private Map<TableName, TableState> tableStates = new HashMap<>();
327  private final RetryCounterFactory lockFileRetryCounterFactory;
328  private final RetryCounterFactory createZNodeRetryCounterFactory;
329
330  private Map<TableName, Set<String>> skippedRegions = new HashMap<>();
331
332  private ZKWatcher zkw = null;
333  private String hbckEphemeralNodePath = null;
334  private boolean hbckZodeCreated = false;
335
336  /**
337   * Constructor
338   *
339   * @param conf Configuration object
340   * @throws MasterNotRunningException if the master is not running
341   * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
342   */
343  public HBaseFsck(Configuration conf) throws IOException, ClassNotFoundException {
344    this(conf, createThreadPool(conf));
345  }
346
347  private static ExecutorService createThreadPool(Configuration conf) {
348    int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
349    return new ScheduledThreadPoolExecutor(numThreads,
350      new ThreadFactoryBuilder().setNameFormat("hbasefsck-pool-%d").setDaemon(true)
351        .setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build());
352  }
353
354  /**
355   * Constructor
356   *
357   * @param conf
358   *          Configuration object
359   * @throws MasterNotRunningException
360   *           if the master is not running
361   * @throws ZooKeeperConnectionException
362   *           if unable to connect to ZooKeeper
363   */
364  public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
365      ZooKeeperConnectionException, IOException, ClassNotFoundException {
366    super(conf);
367    errors = getErrorReporter(getConf());
368    this.executor = exec;
369    lockFileRetryCounterFactory = createLockRetryCounterFactory(getConf());
370    createZNodeRetryCounterFactory = createZnodeRetryCounterFactory(getConf());
371    zkw = createZooKeeperWatcher();
372  }
373
374  /**
375   * @return A retry counter factory configured for retrying lock file creation.
376   */
377  public static RetryCounterFactory createLockRetryCounterFactory(Configuration conf) {
378    return new RetryCounterFactory(
379        conf.getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
380        conf.getInt("hbase.hbck.lockfile.attempt.sleep.interval",
381            DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),
382        conf.getInt("hbase.hbck.lockfile.attempt.maxsleeptime",
383            DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));
384  }
385
386  /**
387   * @return A retry counter factory configured for retrying znode creation.
388   */
389  private static RetryCounterFactory createZnodeRetryCounterFactory(Configuration conf) {
390    return new RetryCounterFactory(
391        conf.getInt("hbase.hbck.createznode.attempts", DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS),
392        conf.getInt("hbase.hbck.createznode.attempt.sleep.interval",
393            DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL),
394        conf.getInt("hbase.hbck.createznode.attempt.maxsleeptime",
395            DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME));
396  }
397
398  /**
399   * @return Return the tmp dir this tool writes too.
400   */
401  @InterfaceAudience.Private
402  public static Path getTmpDir(Configuration conf) throws IOException {
403    return new Path(CommonFSUtils.getRootDir(conf), HConstants.HBASE_TEMP_DIRECTORY);
404  }
405
406  private static class FileLockCallable implements Callable<FSDataOutputStream> {
407    RetryCounter retryCounter;
408    private final Configuration conf;
409    private Path hbckLockPath = null;
410
411    public FileLockCallable(Configuration conf, RetryCounter retryCounter) {
412      this.retryCounter = retryCounter;
413      this.conf = conf;
414    }
415
416    /**
417     * @return Will be <code>null</code> unless you call {@link #call()}
418     */
419    Path getHbckLockPath() {
420      return this.hbckLockPath;
421    }
422
423    @Override
424    public FSDataOutputStream call() throws IOException {
425      try {
426        FileSystem fs = CommonFSUtils.getCurrentFileSystem(this.conf);
427        FsPermission defaultPerms =
428          CommonFSUtils.getFilePermissions(fs, this.conf, HConstants.DATA_FILE_UMASK_KEY);
429        Path tmpDir = getTmpDir(conf);
430        this.hbckLockPath = new Path(tmpDir, HBCK_LOCK_FILE);
431        fs.mkdirs(tmpDir);
432        final FSDataOutputStream out = createFileWithRetries(fs, this.hbckLockPath, defaultPerms);
433        out.writeBytes(InetAddress.getLocalHost().toString());
434        // Add a note into the file we write on why hbase2 is writing out an hbck1 lock file.
435        out.writeBytes(" Written by an hbase-2.x Master to block an " +
436            "attempt by an hbase-1.x HBCK tool making modification to state. " +
437            "See 'HBCK must match HBase server version' in the hbase refguide.");
438        out.flush();
439        return out;
440      } catch(RemoteException e) {
441        if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
442          return null;
443        } else {
444          throw e;
445        }
446      }
447    }
448
449    private FSDataOutputStream createFileWithRetries(final FileSystem fs,
450        final Path hbckLockFilePath, final FsPermission defaultPerms)
451        throws IOException {
452      IOException exception = null;
453      do {
454        try {
455          return CommonFSUtils.create(fs, hbckLockFilePath, defaultPerms, false);
456        } catch (IOException ioe) {
457          LOG.info("Failed to create lock file " + hbckLockFilePath.getName()
458              + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of "
459              + retryCounter.getMaxAttempts());
460          LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(),
461              ioe);
462          try {
463            exception = ioe;
464            retryCounter.sleepUntilNextRetry();
465          } catch (InterruptedException ie) {
466            throw (InterruptedIOException) new InterruptedIOException(
467                "Can't create lock file " + hbckLockFilePath.getName())
468            .initCause(ie);
469          }
470        }
471      } while (retryCounter.shouldRetry());
472
473      throw exception;
474    }
475  }
476
477  /**
478   * This method maintains a lock using a file. If the creation fails we return null
479   *
480   * @return FSDataOutputStream object corresponding to the newly opened lock file
481   * @throws IOException if IO failure occurs
482   */
483  public static Pair<Path, FSDataOutputStream> checkAndMarkRunningHbck(Configuration conf,
484      RetryCounter retryCounter) throws IOException {
485    FileLockCallable callable = new FileLockCallable(conf, retryCounter);
486    ExecutorService executor = Executors.newFixedThreadPool(1);
487    FutureTask<FSDataOutputStream> futureTask = new FutureTask<>(callable);
488    executor.execute(futureTask);
489    final int timeoutInSeconds = conf.getInt(
490      "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT);
491    FSDataOutputStream stream = null;
492    try {
493      stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);
494    } catch (ExecutionException ee) {
495      LOG.warn("Encountered exception when opening lock file", ee);
496    } catch (InterruptedException ie) {
497      LOG.warn("Interrupted when opening lock file", ie);
498      Thread.currentThread().interrupt();
499    } catch (TimeoutException exception) {
500      // took too long to obtain lock
501      LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");
502      futureTask.cancel(true);
503    } finally {
504      executor.shutdownNow();
505    }
506    return new Pair<Path, FSDataOutputStream>(callable.getHbckLockPath(), stream);
507  }
508
509  private void unlockHbck() {
510    if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) {
511      RetryCounter retryCounter = lockFileRetryCounterFactory.create();
512      do {
513        try {
514          IOUtils.closeQuietly(hbckOutFd);
515          CommonFSUtils.delete(CommonFSUtils.getCurrentFileSystem(getConf()), HBCK_LOCK_PATH, true);
516          LOG.info("Finishing hbck");
517          return;
518        } catch (IOException ioe) {
519          LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try="
520              + (retryCounter.getAttemptTimes() + 1) + " of "
521              + retryCounter.getMaxAttempts());
522          LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe);
523          try {
524            retryCounter.sleepUntilNextRetry();
525          } catch (InterruptedException ie) {
526            Thread.currentThread().interrupt();
527            LOG.warn("Interrupted while deleting lock file" +
528                HBCK_LOCK_PATH);
529            return;
530          }
531        }
532      } while (retryCounter.shouldRetry());
533    }
534  }
535
536  /**
537   * To repair region consistency, one must call connect() in order to repair
538   * online state.
539   */
540  public void connect() throws IOException {
541
542    if (isExclusive()) {
543      // Grab the lock
544      Pair<Path, FSDataOutputStream> pair =
545          checkAndMarkRunningHbck(getConf(), this.lockFileRetryCounterFactory.create());
546      HBCK_LOCK_PATH = pair.getFirst();
547      this.hbckOutFd = pair.getSecond();
548      if (hbckOutFd == null) {
549        setRetCode(-1);
550        LOG.error("Another instance of hbck is fixing HBase, exiting this instance. " +
551            "[If you are sure no other instance is running, delete the lock file " +
552            HBCK_LOCK_PATH + " and rerun the tool]");
553        throw new IOException("Duplicate hbck - Abort");
554      }
555
556      // Make sure to cleanup the lock
557      hbckLockCleanup.set(true);
558    }
559
560
561    // Add a shutdown hook to this thread, in case user tries to
562    // kill the hbck with a ctrl-c, we want to cleanup the lock so that
563    // it is available for further calls
564    Runtime.getRuntime().addShutdownHook(new Thread() {
565      @Override
566      public void run() {
567        IOUtils.closeQuietly(HBaseFsck.this);
568        cleanupHbckZnode();
569        unlockHbck();
570      }
571    });
572
573    LOG.info("Launching hbck");
574
575    connection = ConnectionFactory.createConnection(getConf());
576    admin = connection.getAdmin();
577    meta = connection.getTable(TableName.META_TABLE_NAME);
578    status = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS,
579      Option.DEAD_SERVERS, Option.MASTER, Option.BACKUP_MASTERS,
580      Option.REGIONS_IN_TRANSITION, Option.HBASE_VERSION));
581  }
582
583  /**
584   * Get deployed regions according to the region servers.
585   */
586  private void loadDeployedRegions() throws IOException, InterruptedException {
587    // From the master, get a list of all known live region servers
588    Collection<ServerName> regionServers = status.getLiveServerMetrics().keySet();
589    errors.print("Number of live region servers: " + regionServers.size());
590    if (details) {
591      for (ServerName rsinfo: regionServers) {
592        errors.print("  " + rsinfo.getServerName());
593      }
594    }
595
596    // From the master, get a list of all dead region servers
597    Collection<ServerName> deadRegionServers = status.getDeadServerNames();
598    errors.print("Number of dead region servers: " + deadRegionServers.size());
599    if (details) {
600      for (ServerName name: deadRegionServers) {
601        errors.print("  " + name);
602      }
603    }
604
605    // Print the current master name and state
606    errors.print("Master: " + status.getMasterName());
607
608    // Print the list of all backup masters
609    Collection<ServerName> backupMasters = status.getBackupMasterNames();
610    errors.print("Number of backup masters: " + backupMasters.size());
611    if (details) {
612      for (ServerName name: backupMasters) {
613        errors.print("  " + name);
614      }
615    }
616
617    errors.print("Average load: " + status.getAverageLoad());
618    errors.print("Number of requests: " + status.getRequestCount());
619    errors.print("Number of regions: " + status.getRegionCount());
620
621    List<RegionState> rits = status.getRegionStatesInTransition();
622    errors.print("Number of regions in transition: " + rits.size());
623    if (details) {
624      for (RegionState state: rits) {
625        errors.print("  " + state.toDescriptiveString());
626      }
627    }
628
629    // Determine what's deployed
630    processRegionServers(regionServers);
631  }
632
633  /**
634   * Clear the current state of hbck.
635   */
636  private void clearState() {
637    // Make sure regionInfo is empty before starting
638    fixes = 0;
639    regionInfoMap.clear();
640    emptyRegionInfoQualifiers.clear();
641    tableStates.clear();
642    errors.clear();
643    tablesInfo.clear();
644    orphanHdfsDirs.clear();
645    skippedRegions.clear();
646  }
647
648  /**
649   * This repair method analyzes hbase data in hdfs and repairs it to satisfy
650   * the table integrity rules.  HBase doesn't need to be online for this
651   * operation to work.
652   */
653  public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
654    // Initial pass to fix orphans.
655    if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
656        || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
657      LOG.info("Loading regioninfos HDFS");
658      // if nothing is happening this should always complete in two iterations.
659      int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
660      int curIter = 0;
661      do {
662        clearState(); // clears hbck state and reset fixes to 0 and.
663        // repair what's on HDFS
664        restoreHdfsIntegrity();
665        curIter++;// limit the number of iterations.
666      } while (fixes > 0 && curIter <= maxIterations);
667
668      // Repairs should be done in the first iteration and verification in the second.
669      // If there are more than 2 passes, something funny has happened.
670      if (curIter > 2) {
671        if (curIter == maxIterations) {
672          LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
673              + "Tables integrity may not be fully repaired!");
674        } else {
675          LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
676        }
677      }
678    }
679  }
680
681  /**
682   * This repair method requires the cluster to be online since it contacts
683   * region servers and the masters.  It makes each region's state in HDFS, in
684   * hbase:meta, and deployments consistent.
685   *
686   * @return If &gt; 0 , number of errors detected, if &lt; 0 there was an unrecoverable
687   *     error.  If 0, we have a clean hbase.
688   */
689  public int onlineConsistencyRepair() throws IOException, KeeperException,
690    InterruptedException {
691
692    // get regions according to what is online on each RegionServer
693    loadDeployedRegions();
694    // check whether hbase:meta is deployed and online
695    recordMetaRegion();
696    // Check if hbase:meta is found only once and in the right place
697    if (!checkMetaRegion()) {
698      String errorMsg = "hbase:meta table is not consistent. ";
699      if (shouldFixAssignments()) {
700        errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
701      } else {
702        errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
703      }
704      errors.reportError(errorMsg + " Exiting...");
705      return -2;
706    }
707    // Not going with further consistency check for tables when hbase:meta itself is not consistent.
708    LOG.info("Loading regionsinfo from the hbase:meta table");
709    boolean success = loadMetaEntries();
710    if (!success) return -1;
711
712    // Empty cells in hbase:meta?
713    reportEmptyMetaCells();
714
715    // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
716    if (shouldFixEmptyMetaCells()) {
717      fixEmptyMetaCells();
718    }
719
720    // get a list of all tables that have not changed recently.
721    if (!checkMetaOnly) {
722      reportTablesInFlux();
723    }
724
725    // Get disabled tables states
726    loadTableStates();
727
728    // load regiondirs and regioninfos from HDFS
729    if (shouldCheckHdfs()) {
730      LOG.info("Loading region directories from HDFS");
731      loadHdfsRegionDirs();
732      LOG.info("Loading region information from HDFS");
733      loadHdfsRegionInfos();
734    }
735
736    // fix the orphan tables
737    fixOrphanTables();
738
739    LOG.info("Checking and fixing region consistency");
740    // Check and fix consistency
741    checkAndFixConsistency();
742
743    // Check integrity (does not fix)
744    checkIntegrity();
745    return errors.getErrorList().size();
746  }
747
748  /**
749   * This method maintains an ephemeral znode. If the creation fails we return false or throw
750   * exception
751   *
752   * @return true if creating znode succeeds; false otherwise
753   * @throws IOException if IO failure occurs
754   */
755  private boolean setMasterInMaintenanceMode() throws IOException {
756    RetryCounter retryCounter = createZNodeRetryCounterFactory.create();
757    hbckEphemeralNodePath = ZNodePaths.joinZNode(
758      zkw.getZNodePaths().masterMaintZNode,
759      "hbck-" + Long.toString(EnvironmentEdgeManager.currentTime()));
760    do {
761      try {
762        hbckZodeCreated = ZKUtil.createEphemeralNodeAndWatch(zkw, hbckEphemeralNodePath, null);
763        if (hbckZodeCreated) {
764          break;
765        }
766      } catch (KeeperException e) {
767        if (retryCounter.getAttemptTimes() >= retryCounter.getMaxAttempts()) {
768           throw new IOException("Can't create znode " + hbckEphemeralNodePath, e);
769        }
770        // fall through and retry
771      }
772
773      LOG.warn("Fail to create znode " + hbckEphemeralNodePath + ", try=" +
774          (retryCounter.getAttemptTimes() + 1) + " of " + retryCounter.getMaxAttempts());
775
776      try {
777        retryCounter.sleepUntilNextRetry();
778      } catch (InterruptedException ie) {
779        throw (InterruptedIOException) new InterruptedIOException(
780              "Can't create znode " + hbckEphemeralNodePath).initCause(ie);
781      }
782    } while (retryCounter.shouldRetry());
783    return hbckZodeCreated;
784  }
785
786  private void cleanupHbckZnode() {
787    try {
788      if (zkw != null && hbckZodeCreated) {
789        ZKUtil.deleteNode(zkw, hbckEphemeralNodePath);
790        hbckZodeCreated = false;
791      }
792    } catch (KeeperException e) {
793      // Ignore
794      if (!e.code().equals(KeeperException.Code.NONODE)) {
795        LOG.warn("Delete HBCK znode " + hbckEphemeralNodePath + " failed ", e);
796      }
797    }
798  }
799
800  /**
801   * Contacts the master and prints out cluster-wide information
802   * @return 0 on success, non-zero on failure
803   */
804  public int onlineHbck()
805      throws IOException, KeeperException, InterruptedException, ReplicationException {
806    // print hbase server version
807    errors.print("Version: " + status.getHBaseVersion());
808
809    // Clean start
810    clearState();
811    // Do offline check and repair first
812    offlineHdfsIntegrityRepair();
813    offlineReferenceFileRepair();
814    offlineHLinkFileRepair();
815    // If Master runs maintenance tasks (such as balancer, catalog janitor, etc) during online
816    // hbck, it is likely that hbck would be misled and report transient errors.  Therefore, it
817    // is better to set Master into maintenance mode during online hbck.
818    //
819    if (!setMasterInMaintenanceMode()) {
820      LOG.warn("HBCK is running while master is not in maintenance mode, you might see transient "
821        + "error.  Please run HBCK multiple times to reduce the chance of transient error.");
822    }
823
824    onlineConsistencyRepair();
825
826    if (checkRegionBoundaries) {
827      checkRegionBoundaries();
828    }
829
830    checkAndFixReplication();
831
832    cleanReplicationBarrier();
833
834    // Remove the hbck znode
835    cleanupHbckZnode();
836
837    // Remove the hbck lock
838    unlockHbck();
839
840    // Print table summary
841    printTableSummary(tablesInfo);
842    return errors.summarize();
843  }
844
845  public static byte[] keyOnly(byte[] b) {
846    if (b == null)
847      return b;
848    int rowlength = Bytes.toShort(b, 0);
849    byte[] result = new byte[rowlength];
850    System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
851    return result;
852  }
853
854  @Override
855  public void close() throws IOException {
856    try {
857      cleanupHbckZnode();
858      unlockHbck();
859    } catch (Exception io) {
860      LOG.warn(io.toString(), io);
861    } finally {
862      if (zkw != null) {
863        zkw.close();
864        zkw = null;
865      }
866      IOUtils.closeQuietly(admin);
867      IOUtils.closeQuietly(meta);
868      IOUtils.closeQuietly(connection);
869    }
870  }
871
872  private static class RegionBoundariesInformation {
873    public byte [] regionName;
874    public byte [] metaFirstKey;
875    public byte [] metaLastKey;
876    public byte [] storesFirstKey;
877    public byte [] storesLastKey;
878    @Override
879    public String toString () {
880      return "regionName=" + Bytes.toStringBinary(regionName) +
881             "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
882             "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
883             "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
884             "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
885    }
886  }
887
888  public void checkRegionBoundaries() {
889    try {
890      ByteArrayComparator comparator = new ByteArrayComparator();
891      List<RegionInfo> regions = MetaTableAccessor.getAllRegions(connection, true);
892      final RegionBoundariesInformation currentRegionBoundariesInformation =
893          new RegionBoundariesInformation();
894      Path hbaseRoot = CommonFSUtils.getRootDir(getConf());
895      for (RegionInfo regionInfo : regions) {
896        Path tableDir = CommonFSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
897        currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
898        // For each region, get the start and stop key from the META and compare them to the
899        // same information from the Stores.
900        Path path = new Path(tableDir, regionInfo.getEncodedName());
901        FileSystem fs = path.getFileSystem(getConf());
902        FileStatus[] files = fs.listStatus(path);
903        // For all the column families in this region...
904        byte[] storeFirstKey = null;
905        byte[] storeLastKey = null;
906        for (FileStatus file : files) {
907          String fileName = file.getPath().toString();
908          fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
909          if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
910            FileStatus[] storeFiles = fs.listStatus(file.getPath());
911            // For all the stores in this column family.
912            for (FileStatus storeFile : storeFiles) {
913              HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(),
914                CacheConfig.DISABLED, true, getConf());
915              if ((reader.getFirstKey() != null)
916                  && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
917                      ((KeyValue.KeyOnlyKeyValue) reader.getFirstKey().get()).getKey()) > 0))) {
918                storeFirstKey = ((KeyValue.KeyOnlyKeyValue)reader.getFirstKey().get()).getKey();
919              }
920              if ((reader.getLastKey() != null)
921                  && ((storeLastKey == null) || (comparator.compare(storeLastKey,
922                      ((KeyValue.KeyOnlyKeyValue)reader.getLastKey().get()).getKey())) < 0)) {
923                storeLastKey = ((KeyValue.KeyOnlyKeyValue)reader.getLastKey().get()).getKey();
924              }
925              reader.close();
926            }
927          }
928        }
929        currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
930        currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
931        currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
932        currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
933        if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
934          currentRegionBoundariesInformation.metaFirstKey = null;
935        if (currentRegionBoundariesInformation.metaLastKey.length == 0)
936          currentRegionBoundariesInformation.metaLastKey = null;
937
938        // For a region to be correct, we need the META start key to be smaller or equal to the
939        // smallest start key from all the stores, and the start key from the next META entry to
940        // be bigger than the last key from all the current stores. First region start key is null;
941        // Last region end key is null; some regions can be empty and not have any store.
942
943        boolean valid = true;
944        // Checking start key.
945        if ((currentRegionBoundariesInformation.storesFirstKey != null)
946            && (currentRegionBoundariesInformation.metaFirstKey != null)) {
947          valid = valid
948              && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
949                currentRegionBoundariesInformation.metaFirstKey) >= 0;
950        }
951        // Checking stop key.
952        if ((currentRegionBoundariesInformation.storesLastKey != null)
953            && (currentRegionBoundariesInformation.metaLastKey != null)) {
954          valid = valid
955              && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
956                currentRegionBoundariesInformation.metaLastKey) < 0;
957        }
958        if (!valid) {
959          errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
960            tablesInfo.get(regionInfo.getTable()));
961          LOG.warn("Region's boundaries not aligned between stores and META for:");
962          LOG.warn(Objects.toString(currentRegionBoundariesInformation));
963        }
964      }
965    } catch (IOException e) {
966      LOG.error(e.toString(), e);
967    }
968  }
969
970  /**
971   * Iterates through the list of all orphan/invalid regiondirs.
972   */
973  private void adoptHdfsOrphans(Collection<HbckRegionInfo> orphanHdfsDirs) throws IOException {
974    for (HbckRegionInfo hi : orphanHdfsDirs) {
975      LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
976      adoptHdfsOrphan(hi);
977    }
978  }
979
980  /**
981   * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
982   * these orphans by creating a new region, and moving the column families,
983   * recovered edits, WALs, into the new region dir.  We determine the region
984   * startkey and endkeys by looking at all of the hfiles inside the column
985   * families to identify the min and max keys. The resulting region will
986   * likely violate table integrity but will be dealt with by merging
987   * overlapping regions.
988   */
989  @SuppressWarnings("deprecation")
990  private void adoptHdfsOrphan(HbckRegionInfo hi) throws IOException {
991    Path p = hi.getHdfsRegionDir();
992    FileSystem fs = p.getFileSystem(getConf());
993    FileStatus[] dirs = fs.listStatus(p);
994    if (dirs == null) {
995      LOG.warn("Attempt to adopt orphan hdfs region skipped because no files present in " +
996          p + ". This dir could probably be deleted.");
997      return ;
998    }
999
1000    TableName tableName = hi.getTableName();
1001    HbckTableInfo tableInfo = tablesInfo.get(tableName);
1002    Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
1003    TableDescriptor template = tableInfo.getTableDescriptor();
1004
1005    // find min and max key values
1006    Pair<byte[],byte[]> orphanRegionRange = null;
1007    for (FileStatus cf : dirs) {
1008      String cfName= cf.getPath().getName();
1009      // TODO Figure out what the special dirs are
1010      if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
1011
1012      FileStatus[] hfiles = fs.listStatus(cf.getPath());
1013      for (FileStatus hfile : hfiles) {
1014        byte[] start, end;
1015        HFile.Reader hf = null;
1016        try {
1017          hf = HFile.createReader(fs, hfile.getPath(), CacheConfig.DISABLED, true, getConf());
1018          Optional<Cell> startKv = hf.getFirstKey();
1019          start = CellUtil.cloneRow(startKv.get());
1020          Optional<Cell> endKv = hf.getLastKey();
1021          end = CellUtil.cloneRow(endKv.get());
1022        } catch (IOException ioe) {
1023          LOG.warn("Problem reading orphan file " + hfile + ", skipping");
1024          continue;
1025        } catch (NullPointerException ioe) {
1026          LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
1027          continue;
1028        } finally {
1029          if (hf != null) {
1030            hf.close();
1031          }
1032        }
1033
1034        // expand the range to include the range of all hfiles
1035        if (orphanRegionRange == null) {
1036          // first range
1037          orphanRegionRange = new Pair<>(start, end);
1038        } else {
1039          // TODO add test
1040
1041          // expand range only if the hfile is wider.
1042          if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
1043            orphanRegionRange.setFirst(start);
1044          }
1045          if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
1046            orphanRegionRange.setSecond(end);
1047          }
1048        }
1049      }
1050    }
1051    if (orphanRegionRange == null) {
1052      LOG.warn("No data in dir " + p + ", sidelining data");
1053      fixes++;
1054      sidelineRegionDir(fs, hi);
1055      return;
1056    }
1057    LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
1058        Bytes.toString(orphanRegionRange.getSecond()) + ")");
1059
1060    // create new region on hdfs. move data into place.
1061    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(template.getTableName())
1062        .setStartKey(orphanRegionRange.getFirst())
1063        .setEndKey(Bytes.add(orphanRegionRange.getSecond(), new byte[1]))
1064        .build();
1065    LOG.info("Creating new region : " + regionInfo);
1066    HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), regionInfo, template);
1067    Path target = region.getRegionFileSystem().getRegionDir();
1068
1069    // rename all the data to new region
1070    mergeRegionDirs(target, hi);
1071    fixes++;
1072  }
1073
1074  /**
1075   * This method determines if there are table integrity errors in HDFS.  If
1076   * there are errors and the appropriate "fix" options are enabled, the method
1077   * will first correct orphan regions making them into legit regiondirs, and
1078   * then reload to merge potentially overlapping regions.
1079   *
1080   * @return number of table integrity errors found
1081   */
1082  private int restoreHdfsIntegrity() throws IOException, InterruptedException {
1083    // Determine what's on HDFS
1084    LOG.info("Loading HBase regioninfo from HDFS...");
1085    loadHdfsRegionDirs(); // populating regioninfo table.
1086
1087    int errs = errors.getErrorList().size();
1088    // First time just get suggestions.
1089    tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1090    checkHdfsIntegrity(false, false);
1091
1092    if (errors.getErrorList().size() == errs) {
1093      LOG.info("No integrity errors.  We are done with this phase. Glorious.");
1094      return 0;
1095    }
1096
1097    if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
1098      adoptHdfsOrphans(orphanHdfsDirs);
1099      // TODO optimize by incrementally adding instead of reloading.
1100    }
1101
1102    // Make sure there are no holes now.
1103    if (shouldFixHdfsHoles()) {
1104      clearState(); // this also resets # fixes.
1105      loadHdfsRegionDirs();
1106      tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1107      tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
1108    }
1109
1110    // Now we fix overlaps
1111    if (shouldFixHdfsOverlaps()) {
1112      // second pass we fix overlaps.
1113      clearState(); // this also resets # fixes.
1114      loadHdfsRegionDirs();
1115      tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1116      tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
1117    }
1118
1119    return errors.getErrorList().size();
1120  }
1121
1122  /**
1123   * Scan all the store file names to find any lingering reference files,
1124   * which refer to some none-exiting files. If "fix" option is enabled,
1125   * any lingering reference file will be sidelined if found.
1126   * <p>
1127   * Lingering reference file prevents a region from opening. It has to
1128   * be fixed before a cluster can start properly.
1129   */
1130  private void offlineReferenceFileRepair() throws IOException, InterruptedException {
1131    clearState();
1132    Configuration conf = getConf();
1133    Path hbaseRoot = CommonFSUtils.getRootDir(conf);
1134    FileSystem fs = hbaseRoot.getFileSystem(conf);
1135    LOG.info("Computing mapping of all store files");
1136    Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot,
1137      new FSUtils.ReferenceFileFilter(fs), executor, errors);
1138    errors.print("");
1139    LOG.info("Validating mapping using HDFS state");
1140    for (Path path: allFiles.values()) {
1141      Path referredToFile = StoreFileInfo.getReferredToFile(path);
1142      if (fs.exists(referredToFile)) continue;  // good, expected
1143
1144      // Found a lingering reference file
1145      errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
1146        "Found lingering reference file " + path);
1147      if (!shouldFixReferenceFiles()) continue;
1148
1149      // Now, trying to fix it since requested
1150      boolean success = false;
1151      String pathStr = path.toString();
1152
1153      // A reference file path should be like
1154      // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
1155      // Up 5 directories to get the root folder.
1156      // So the file will be sidelined to a similar folder structure.
1157      int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
1158      for (int i = 0; index > 0 && i < 5; i++) {
1159        index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
1160      }
1161      if (index > 0) {
1162        Path rootDir = getSidelineDir();
1163        Path dst = new Path(rootDir, pathStr.substring(index + 1));
1164        fs.mkdirs(dst.getParent());
1165        LOG.info("Trying to sideline reference file "
1166          + path + " to " + dst);
1167        setShouldRerun();
1168
1169        success = fs.rename(path, dst);
1170        debugLsr(dst);
1171
1172      }
1173      if (!success) {
1174        LOG.error("Failed to sideline reference file " + path);
1175      }
1176    }
1177  }
1178
1179  /**
1180   * Scan all the store file names to find any lingering HFileLink files,
1181   * which refer to some none-exiting files. If "fix" option is enabled,
1182   * any lingering HFileLink file will be sidelined if found.
1183   */
1184  private void offlineHLinkFileRepair() throws IOException, InterruptedException {
1185    Configuration conf = getConf();
1186    Path hbaseRoot = CommonFSUtils.getRootDir(conf);
1187    FileSystem fs = hbaseRoot.getFileSystem(conf);
1188    LOG.info("Computing mapping of all link files");
1189    Map<String, Path> allFiles = FSUtils
1190        .getTableStoreFilePathMap(fs, hbaseRoot, new FSUtils.HFileLinkFilter(), executor, errors);
1191    errors.print("");
1192
1193    LOG.info("Validating mapping using HDFS state");
1194    for (Path path : allFiles.values()) {
1195      // building HFileLink object to gather locations
1196      HFileLink actualLink = HFileLink.buildFromHFileLinkPattern(conf, path);
1197      if (actualLink.exists(fs)) continue; // good, expected
1198
1199      // Found a lingering HFileLink
1200      errors.reportError(ERROR_CODE.LINGERING_HFILELINK, "Found lingering HFileLink " + path);
1201      if (!shouldFixHFileLinks()) continue;
1202
1203      // Now, trying to fix it since requested
1204      setShouldRerun();
1205
1206      // An HFileLink path should be like
1207      // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/linkedtable=linkedregionname-linkedhfilename
1208      // sidelineing will happen in the ${hbase.rootdir}/${sidelinedir} directory with the same folder structure.
1209      boolean success = sidelineFile(fs, hbaseRoot, path);
1210
1211      if (!success) {
1212        LOG.error("Failed to sideline HFileLink file " + path);
1213      }
1214
1215      // An HFileLink backreference path should be like
1216      // ${hbase.rootdir}/archive/data/namespace/table_name/region_id/family_name/.links-linkedhfilename
1217      // sidelineing will happen in the ${hbase.rootdir}/${sidelinedir} directory with the same folder structure.
1218      Path backRefPath = FileLink.getBackReferencesDir(HFileArchiveUtil
1219              .getStoreArchivePath(conf, HFileLink.getReferencedTableName(path.getName().toString()),
1220                  HFileLink.getReferencedRegionName(path.getName().toString()),
1221                  path.getParent().getName()),
1222          HFileLink.getReferencedHFileName(path.getName().toString()));
1223      success = sidelineFile(fs, hbaseRoot, backRefPath);
1224
1225      if (!success) {
1226        LOG.error("Failed to sideline HFileLink backreference file " + path);
1227      }
1228    }
1229  }
1230
1231  private boolean sidelineFile(FileSystem fs, Path hbaseRoot, Path path) throws IOException {
1232    URI uri = hbaseRoot.toUri().relativize(path.toUri());
1233    if (uri.isAbsolute()) return false;
1234    String relativePath = uri.getPath();
1235    Path rootDir = getSidelineDir();
1236    Path dst = new Path(rootDir, relativePath);
1237    boolean pathCreated = fs.mkdirs(dst.getParent());
1238    if (!pathCreated) {
1239      LOG.error("Failed to create path: " + dst.getParent());
1240      return false;
1241    }
1242    LOG.info("Trying to sideline file " + path + " to " + dst);
1243    return fs.rename(path, dst);
1244  }
1245
1246  /**
1247   * TODO -- need to add tests for this.
1248   */
1249  private void reportEmptyMetaCells() {
1250    errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
1251      emptyRegionInfoQualifiers.size());
1252    if (details) {
1253      for (Result r: emptyRegionInfoQualifiers) {
1254        errors.print("  " + r);
1255      }
1256    }
1257  }
1258
1259  /**
1260   * TODO -- need to add tests for this.
1261   */
1262  private void reportTablesInFlux() {
1263    AtomicInteger numSkipped = new AtomicInteger(0);
1264    TableDescriptor[] allTables = getTables(numSkipped);
1265    errors.print("Number of Tables: " + allTables.length);
1266    if (details) {
1267      if (numSkipped.get() > 0) {
1268        errors.detail("Number of Tables in flux: " + numSkipped.get());
1269      }
1270      for (TableDescriptor td : allTables) {
1271        errors.detail("  Table: " + td.getTableName() + "\t" +
1272                           (td.isReadOnly() ? "ro" : "rw") + "\t" +
1273                            (td.isMetaRegion() ? "META" : "    ") + "\t" +
1274                           " families: " + td.getColumnFamilyCount());
1275      }
1276    }
1277  }
1278
1279  public HbckErrorReporter getErrors() {
1280    return errors;
1281  }
1282
1283  /**
1284   * Populate hbi's from regionInfos loaded from file system.
1285   */
1286  private SortedMap<TableName, HbckTableInfo> loadHdfsRegionInfos()
1287      throws IOException, InterruptedException {
1288    tablesInfo.clear(); // regenerating the data
1289    // generate region split structure
1290    Collection<HbckRegionInfo> hbckRegionInfos = regionInfoMap.values();
1291
1292    // Parallelized read of .regioninfo files.
1293    List<WorkItemHdfsRegionInfo> hbis = new ArrayList<>(hbckRegionInfos.size());
1294    List<Future<Void>> hbiFutures;
1295
1296    for (HbckRegionInfo hbi : hbckRegionInfos) {
1297      WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1298      hbis.add(work);
1299    }
1300
1301    // Submit and wait for completion
1302    hbiFutures = executor.invokeAll(hbis);
1303
1304    for(int i=0; i<hbiFutures.size(); i++) {
1305      WorkItemHdfsRegionInfo work = hbis.get(i);
1306      Future<Void> f = hbiFutures.get(i);
1307      try {
1308        f.get();
1309      } catch(ExecutionException e) {
1310        LOG.warn("Failed to read .regioninfo file for region " +
1311              work.hbi.getRegionNameAsString(), e.getCause());
1312      }
1313    }
1314
1315    Path hbaseRoot = CommonFSUtils.getRootDir(getConf());
1316    FileSystem fs = hbaseRoot.getFileSystem(getConf());
1317    // serialized table info gathering.
1318    for (HbckRegionInfo hbi: hbckRegionInfos) {
1319
1320      if (hbi.getHdfsHRI() == null) {
1321        // was an orphan
1322        continue;
1323      }
1324
1325
1326      // get table name from hdfs, populate various HBaseFsck tables.
1327      TableName tableName = hbi.getTableName();
1328      if (tableName == null) {
1329        // There was an entry in hbase:meta not in the HDFS?
1330        LOG.warn("tableName was null for: " + hbi);
1331        continue;
1332      }
1333
1334      HbckTableInfo modTInfo = tablesInfo.get(tableName);
1335      if (modTInfo == null) {
1336        // only executed once per table.
1337        modTInfo = new HbckTableInfo(tableName, this);
1338        tablesInfo.put(tableName, modTInfo);
1339        try {
1340          TableDescriptor htd =
1341              FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1342          modTInfo.htds.add(htd);
1343        } catch (IOException ioe) {
1344          if (!orphanTableDirs.containsKey(tableName)) {
1345            LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1346            //should only report once for each table
1347            errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1348                "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1349            Set<String> columns = new HashSet<>();
1350            orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1351          }
1352        }
1353      }
1354      if (!hbi.isSkipChecks()) {
1355        modTInfo.addRegionInfo(hbi);
1356      }
1357    }
1358
1359    loadTableInfosForTablesWithNoRegion();
1360    errors.print("");
1361
1362    return tablesInfo;
1363  }
1364
1365  /**
1366   * To get the column family list according to the column family dirs
1367   * @param columns
1368   * @param hbi
1369   * @return a set of column families
1370   * @throws IOException
1371   */
1372  private Set<String> getColumnFamilyList(Set<String> columns, HbckRegionInfo hbi)
1373      throws IOException {
1374    Path regionDir = hbi.getHdfsRegionDir();
1375    FileSystem fs = regionDir.getFileSystem(getConf());
1376    FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1377    for (FileStatus subdir : subDirs) {
1378      String columnfamily = subdir.getPath().getName();
1379      columns.add(columnfamily);
1380    }
1381    return columns;
1382  }
1383
1384  /**
1385   * To fabricate a .tableinfo file with following contents<br>
1386   * 1. the correct tablename <br>
1387   * 2. the correct colfamily list<br>
1388   * 3. the default properties for both {@link TableDescriptor} and {@link ColumnFamilyDescriptor}<br>
1389   * @throws IOException
1390   */
1391  private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1392      Set<String> columns) throws IOException {
1393    if (columns ==null || columns.isEmpty()) return false;
1394    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
1395    for (String columnfamimly : columns) {
1396      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(columnfamimly));
1397    }
1398    fstd.createTableDescriptor(builder.build(), true);
1399    return true;
1400  }
1401
1402  /**
1403   * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1404   * @throws IOException
1405   */
1406  public void fixEmptyMetaCells() throws IOException {
1407    if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1408      LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1409      for (Result region : emptyRegionInfoQualifiers) {
1410        deleteMetaRegion(region.getRow());
1411        errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1412      }
1413      emptyRegionInfoQualifiers.clear();
1414    }
1415  }
1416
1417  /**
1418   * To fix orphan table by creating a .tableinfo file under tableDir <br>
1419   * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1420   * 2. else create a default .tableinfo file with following items<br>
1421   * &nbsp;2.1 the correct tablename <br>
1422   * &nbsp;2.2 the correct colfamily list<br>
1423   * &nbsp;2.3 the default properties for both {@link TableDescriptor} and {@link ColumnFamilyDescriptor}<br>
1424   * @throws IOException
1425   */
1426  public void fixOrphanTables() throws IOException {
1427    if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1428
1429      List<TableName> tmpList = new ArrayList<>(orphanTableDirs.keySet().size());
1430      tmpList.addAll(orphanTableDirs.keySet());
1431      TableDescriptor[] htds = getTableDescriptors(tmpList);
1432      Iterator<Entry<TableName, Set<String>>> iter =
1433          orphanTableDirs.entrySet().iterator();
1434      int j = 0;
1435      int numFailedCase = 0;
1436      FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1437      while (iter.hasNext()) {
1438        Entry<TableName, Set<String>> entry =
1439            iter.next();
1440        TableName tableName = entry.getKey();
1441        LOG.info("Trying to fix orphan table error: " + tableName);
1442        if (j < htds.length) {
1443          if (tableName.equals(htds[j].getTableName())) {
1444            TableDescriptor htd = htds[j];
1445            LOG.info("fixing orphan table: " + tableName + " from cache");
1446            fstd.createTableDescriptor(htd, true);
1447            j++;
1448            iter.remove();
1449          }
1450        } else {
1451          if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1452            LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1453            LOG.warn("Strongly recommend to modify the TableDescriptor if necessary for: " + tableName);
1454            iter.remove();
1455          } else {
1456            LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1457            numFailedCase++;
1458          }
1459        }
1460        fixes++;
1461      }
1462
1463      if (orphanTableDirs.isEmpty()) {
1464        // all orphanTableDirs are luckily recovered
1465        // re-run doFsck after recovering the .tableinfo file
1466        setShouldRerun();
1467        LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1468      } else if (numFailedCase > 0) {
1469        LOG.error("Failed to fix " + numFailedCase
1470            + " OrphanTables with default .tableinfo files");
1471      }
1472
1473    }
1474    //cleanup the list
1475    orphanTableDirs.clear();
1476
1477  }
1478
1479  /**
1480   * Log an appropriate message about whether or not overlapping merges are computed in parallel.
1481   */
1482  private void logParallelMerge() {
1483    if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
1484      LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
1485          " false to run serially.");
1486    } else {
1487      LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
1488          " true to run in parallel.");
1489    }
1490  }
1491
1492  private SortedMap<TableName, HbckTableInfo> checkHdfsIntegrity(boolean fixHoles,
1493      boolean fixOverlaps) throws IOException {
1494    LOG.info("Checking HBase region split map from HDFS data...");
1495    logParallelMerge();
1496    for (HbckTableInfo tInfo : tablesInfo.values()) {
1497      TableIntegrityErrorHandler handler;
1498      if (fixHoles || fixOverlaps) {
1499        handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1500          fixHoles, fixOverlaps);
1501      } else {
1502        handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1503      }
1504      if (!tInfo.checkRegionChain(handler)) {
1505        // should dump info as well.
1506        errors.report("Found inconsistency in table " + tInfo.getName());
1507      }
1508    }
1509    return tablesInfo;
1510  }
1511
1512  Path getSidelineDir() throws IOException {
1513    if (sidelineDir == null) {
1514      Path hbaseDir = CommonFSUtils.getRootDir(getConf());
1515      Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1516      sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1517          + startMillis);
1518    }
1519    return sidelineDir;
1520  }
1521
1522  /**
1523   * Sideline a region dir (instead of deleting it)
1524   */
1525  Path sidelineRegionDir(FileSystem fs, HbckRegionInfo hi) throws IOException {
1526    return sidelineRegionDir(fs, null, hi);
1527  }
1528
1529  /**
1530   * Sideline a region dir (instead of deleting it)
1531   *
1532   * @param parentDir if specified, the region will be sidelined to folder like
1533   *     {@literal .../parentDir/<table name>/<region name>}. The purpose is to group together
1534   *     similar regions sidelined, for example, those regions should be bulk loaded back later
1535   *     on. If NULL, it is ignored.
1536   */
1537  Path sidelineRegionDir(FileSystem fs,
1538      String parentDir, HbckRegionInfo hi) throws IOException {
1539    TableName tableName = hi.getTableName();
1540    Path regionDir = hi.getHdfsRegionDir();
1541
1542    if (!fs.exists(regionDir)) {
1543      LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1544      return null;
1545    }
1546
1547    Path rootDir = getSidelineDir();
1548    if (parentDir != null) {
1549      rootDir = new Path(rootDir, parentDir);
1550    }
1551    Path sidelineTableDir= CommonFSUtils.getTableDir(rootDir, tableName);
1552    Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1553    fs.mkdirs(sidelineRegionDir);
1554    boolean success = false;
1555    FileStatus[] cfs =  fs.listStatus(regionDir);
1556    if (cfs == null) {
1557      LOG.info("Region dir is empty: " + regionDir);
1558    } else {
1559      for (FileStatus cf : cfs) {
1560        Path src = cf.getPath();
1561        Path dst =  new Path(sidelineRegionDir, src.getName());
1562        if (fs.isFile(src)) {
1563          // simple file
1564          success = fs.rename(src, dst);
1565          if (!success) {
1566            String msg = "Unable to rename file " + src +  " to " + dst;
1567            LOG.error(msg);
1568            throw new IOException(msg);
1569          }
1570          continue;
1571        }
1572
1573        // is a directory.
1574        fs.mkdirs(dst);
1575
1576        LOG.info("Sidelining files from " + src + " into containing region " + dst);
1577        // FileSystem.rename is inconsistent with directories -- if the
1578        // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1579        // it moves the src into the dst dir resulting in (foo/a/b).  If
1580        // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1581        FileStatus[] hfiles = fs.listStatus(src);
1582        if (hfiles != null && hfiles.length > 0) {
1583          for (FileStatus hfile : hfiles) {
1584            success = fs.rename(hfile.getPath(), dst);
1585            if (!success) {
1586              String msg = "Unable to rename file " + src +  " to " + dst;
1587              LOG.error(msg);
1588              throw new IOException(msg);
1589            }
1590          }
1591        }
1592        LOG.debug("Sideline directory contents:");
1593        debugLsr(sidelineRegionDir);
1594      }
1595    }
1596
1597    LOG.info("Removing old region dir: " + regionDir);
1598    success = fs.delete(regionDir, true);
1599    if (!success) {
1600      String msg = "Unable to delete dir " + regionDir;
1601      LOG.error(msg);
1602      throw new IOException(msg);
1603    }
1604    return sidelineRegionDir;
1605  }
1606
1607  /**
1608   * Load the list of disabled tables in ZK into local set.
1609   * @throws ZooKeeperConnectionException
1610   * @throws IOException
1611   */
1612  private void loadTableStates()
1613  throws IOException {
1614    tableStates = MetaTableAccessor.getTableStates(connection);
1615    // Add hbase:meta so this tool keeps working. In hbase2, meta is always enabled though it
1616    // has no entry in the table states. HBCK doesn't work right w/ hbase2 but just do this in
1617    // meantime.
1618    this.tableStates.put(TableName.META_TABLE_NAME,
1619        new TableState(TableName.META_TABLE_NAME, TableState.State.ENABLED));
1620  }
1621
1622  /**
1623   * Check if the specified region's table is disabled.
1624   * @param tableName table to check status of
1625   */
1626  boolean isTableDisabled(TableName tableName) {
1627    return tableStates.containsKey(tableName)
1628        && tableStates.get(tableName)
1629        .inStates(TableState.State.DISABLED, TableState.State.DISABLING);
1630  }
1631
1632  /**
1633   * Scan HDFS for all regions, recording their information into
1634   * regionInfoMap
1635   */
1636  public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1637    Path rootDir = CommonFSUtils.getRootDir(getConf());
1638    FileSystem fs = rootDir.getFileSystem(getConf());
1639
1640    // list all tables from HDFS
1641    List<FileStatus> tableDirs = Lists.newArrayList();
1642
1643    boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1644
1645    List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1646    for (Path path : paths) {
1647      TableName tableName = CommonFSUtils.getTableName(path);
1648       if ((!checkMetaOnly &&
1649           isTableIncluded(tableName)) ||
1650           tableName.equals(TableName.META_TABLE_NAME)) {
1651         tableDirs.add(fs.getFileStatus(path));
1652       }
1653    }
1654
1655    // verify that version file exists
1656    if (!foundVersionFile) {
1657      errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1658          "Version file does not exist in root dir " + rootDir);
1659      if (shouldFixVersionFile()) {
1660        LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1661            + " file.");
1662        setShouldRerun();
1663        FSUtils.setVersion(fs, rootDir, getConf().getInt(
1664            HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1665            HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1666            HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1667      }
1668    }
1669
1670    // Avoid multithreading at table-level because already multithreaded internally at
1671    // region-level.  Additionally multithreading at table-level can lead to deadlock
1672    // if there are many tables in the cluster.  Since there are a limited # of threads
1673    // in the executor's thread pool and if we multithread at the table-level by putting
1674    // WorkItemHdfsDir callables into the executor, then we will have some threads in the
1675    // executor tied up solely in waiting for the tables' region-level calls to complete.
1676    // If there are enough tables then there will be no actual threads in the pool left
1677    // for the region-level callables to be serviced.
1678    for (FileStatus tableDir : tableDirs) {
1679      LOG.debug("Loading region dirs from " +tableDir.getPath());
1680      WorkItemHdfsDir item = new WorkItemHdfsDir(fs, errors, tableDir);
1681      try {
1682        item.call();
1683      } catch (ExecutionException e) {
1684        LOG.warn("Could not completely load table dir " +
1685            tableDir.getPath(), e.getCause());
1686      }
1687    }
1688    errors.print("");
1689  }
1690
1691  /**
1692   * Record the location of the hbase:meta region as found in ZooKeeper.
1693   */
1694  private boolean recordMetaRegion() throws IOException {
1695    List<HRegionLocation> locs;
1696    try (RegionLocator locator = connection.getRegionLocator(TableName.META_TABLE_NAME)) {
1697      locs = locator.getRegionLocations(HConstants.EMPTY_START_ROW, true);
1698    }
1699    if (locs == null || locs.isEmpty()) {
1700      errors.reportError(ERROR_CODE.NULL_META_REGION, "META region was not found in ZooKeeper");
1701      return false;
1702    }
1703    for (HRegionLocation metaLocation : locs) {
1704      // Check if Meta region is valid and existing
1705      if (metaLocation == null) {
1706        errors.reportError(ERROR_CODE.NULL_META_REGION, "META region location is null");
1707        return false;
1708      }
1709      if (metaLocation.getRegion() == null) {
1710        errors.reportError(ERROR_CODE.NULL_META_REGION, "META location regionInfo is null");
1711        return false;
1712      }
1713      if (metaLocation.getHostname() == null) {
1714        errors.reportError(ERROR_CODE.NULL_META_REGION, "META location hostName is null");
1715        return false;
1716      }
1717      ServerName sn = metaLocation.getServerName();
1718      HbckRegionInfo.MetaEntry m = new HbckRegionInfo.MetaEntry(metaLocation.getRegion(), sn,
1719          EnvironmentEdgeManager.currentTime());
1720      HbckRegionInfo hbckRegionInfo = regionInfoMap.get(metaLocation.getRegion().getEncodedName());
1721      if (hbckRegionInfo == null) {
1722        regionInfoMap.put(metaLocation.getRegion().getEncodedName(), new HbckRegionInfo(m));
1723      } else {
1724        hbckRegionInfo.setMetaEntry(m);
1725      }
1726    }
1727    return true;
1728  }
1729
1730  private ZKWatcher createZooKeeperWatcher() throws IOException {
1731    return new ZKWatcher(getConf(), "hbase Fsck", new Abortable() {
1732      @Override
1733      public void abort(String why, Throwable e) {
1734        LOG.error(why, e);
1735        System.exit(1);
1736      }
1737
1738      @Override
1739      public boolean isAborted() {
1740        return false;
1741      }
1742
1743    });
1744  }
1745
1746  /**
1747   * Contacts each regionserver and fetches metadata about regions.
1748   * @param regionServerList - the list of region servers to connect to
1749   * @throws IOException if a remote or network exception occurs
1750   */
1751  void processRegionServers(Collection<ServerName> regionServerList)
1752    throws IOException, InterruptedException {
1753
1754    List<WorkItemRegion> workItems = new ArrayList<>(regionServerList.size());
1755    List<Future<Void>> workFutures;
1756
1757    // loop to contact each region server in parallel
1758    for (ServerName rsinfo: regionServerList) {
1759      workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1760    }
1761
1762    workFutures = executor.invokeAll(workItems);
1763
1764    for(int i=0; i<workFutures.size(); i++) {
1765      WorkItemRegion item = workItems.get(i);
1766      Future<Void> f = workFutures.get(i);
1767      try {
1768        f.get();
1769      } catch(ExecutionException e) {
1770        LOG.warn("Could not process regionserver {}", item.rsinfo.getAddress(),
1771            e.getCause());
1772      }
1773    }
1774  }
1775
1776  /**
1777   * Check consistency of all regions that have been found in previous phases.
1778   */
1779  private void checkAndFixConsistency()
1780  throws IOException, KeeperException, InterruptedException {
1781    // Divide the checks in two phases. One for default/primary replicas and another
1782    // for the non-primary ones. Keeps code cleaner this way.
1783
1784    List<CheckRegionConsistencyWorkItem> workItems = new ArrayList<>(regionInfoMap.size());
1785    for (java.util.Map.Entry<String, HbckRegionInfo> e: regionInfoMap.entrySet()) {
1786      if (e.getValue().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
1787        workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1788      }
1789    }
1790    checkRegionConsistencyConcurrently(workItems);
1791
1792    boolean prevHdfsCheck = shouldCheckHdfs();
1793    setCheckHdfs(false); //replicas don't have any hdfs data
1794    // Run a pass over the replicas and fix any assignment issues that exist on the currently
1795    // deployed/undeployed replicas.
1796    List<CheckRegionConsistencyWorkItem> replicaWorkItems = new ArrayList<>(regionInfoMap.size());
1797    for (java.util.Map.Entry<String, HbckRegionInfo> e: regionInfoMap.entrySet()) {
1798      if (e.getValue().getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
1799        replicaWorkItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1800      }
1801    }
1802    checkRegionConsistencyConcurrently(replicaWorkItems);
1803    setCheckHdfs(prevHdfsCheck);
1804
1805    // If some regions is skipped during checkRegionConsistencyConcurrently() phase, we might
1806    // not get accurate state of the hbase if continuing. The config here allows users to tune
1807    // the tolerance of number of skipped region.
1808    // TODO: evaluate the consequence to continue the hbck operation without config.
1809    int terminateThreshold =  getConf().getInt("hbase.hbck.skipped.regions.limit", 0);
1810    int numOfSkippedRegions = skippedRegions.size();
1811    if (numOfSkippedRegions > 0 && numOfSkippedRegions > terminateThreshold) {
1812      throw new IOException(numOfSkippedRegions
1813        + " region(s) could not be checked or repaired.  See logs for detail.");
1814    }
1815
1816    if (shouldCheckHdfs()) {
1817      checkAndFixTableStates();
1818    }
1819  }
1820
1821  /**
1822   * Check consistency of all regions using mulitple threads concurrently.
1823   */
1824  private void checkRegionConsistencyConcurrently(
1825    final List<CheckRegionConsistencyWorkItem> workItems)
1826    throws IOException, KeeperException, InterruptedException {
1827    if (workItems.isEmpty()) {
1828      return;  // nothing to check
1829    }
1830
1831    List<Future<Void>> workFutures = executor.invokeAll(workItems);
1832    for(Future<Void> f: workFutures) {
1833      try {
1834        f.get();
1835      } catch(ExecutionException e1) {
1836        LOG.warn("Could not check region consistency " , e1.getCause());
1837        if (e1.getCause() instanceof IOException) {
1838          throw (IOException)e1.getCause();
1839        } else if (e1.getCause() instanceof KeeperException) {
1840          throw (KeeperException)e1.getCause();
1841        } else if (e1.getCause() instanceof InterruptedException) {
1842          throw (InterruptedException)e1.getCause();
1843        } else {
1844          throw new IOException(e1.getCause());
1845        }
1846      }
1847    }
1848  }
1849
1850  class CheckRegionConsistencyWorkItem implements Callable<Void> {
1851    private final String key;
1852    private final HbckRegionInfo hbi;
1853
1854    CheckRegionConsistencyWorkItem(String key, HbckRegionInfo hbi) {
1855      this.key = key;
1856      this.hbi = hbi;
1857    }
1858
1859    @Override
1860    public synchronized Void call() throws Exception {
1861      try {
1862        checkRegionConsistency(key, hbi);
1863      } catch (Exception e) {
1864        // If the region is non-META region, skip this region and send warning/error message; if
1865        // the region is META region, we should not continue.
1866        LOG.warn("Unable to complete check or repair the region '" + hbi.getRegionNameAsString()
1867          + "'.", e);
1868        if (hbi.getHdfsHRI().isMetaRegion()) {
1869          throw e;
1870        }
1871        LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'");
1872        addSkippedRegion(hbi);
1873      }
1874      return null;
1875    }
1876  }
1877
1878  private void addSkippedRegion(final HbckRegionInfo hbi) {
1879    Set<String> skippedRegionNames = skippedRegions.get(hbi.getTableName());
1880    if (skippedRegionNames == null) {
1881      skippedRegionNames = new HashSet<>();
1882    }
1883    skippedRegionNames.add(hbi.getRegionNameAsString());
1884    skippedRegions.put(hbi.getTableName(), skippedRegionNames);
1885  }
1886
1887  /**
1888   * Check and fix table states, assumes full info available:
1889   * - tableInfos
1890   * - empty tables loaded
1891   */
1892  private void checkAndFixTableStates() throws IOException {
1893    // first check dangling states
1894    for (Entry<TableName, TableState> entry : tableStates.entrySet()) {
1895      TableName tableName = entry.getKey();
1896      TableState tableState = entry.getValue();
1897      HbckTableInfo tableInfo = tablesInfo.get(tableName);
1898      if (isTableIncluded(tableName)
1899          && !tableName.isSystemTable()
1900          && tableInfo == null) {
1901        if (fixMeta) {
1902          MetaTableAccessor.deleteTableState(connection, tableName);
1903          TableState state = MetaTableAccessor.getTableState(connection, tableName);
1904          if (state != null) {
1905            errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,
1906                tableName + " unable to delete dangling table state " + tableState);
1907          }
1908        } else if (!checkMetaOnly) {
1909          // dangling table state in meta if checkMetaOnly is false. If checkMetaOnly is
1910          // true, tableInfo will be null as tablesInfo are not polulated for all tables from hdfs
1911          errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,
1912              tableName + " has dangling table state " + tableState);
1913        }
1914      }
1915    }
1916    // check that all tables have states
1917    for (TableName tableName : tablesInfo.keySet()) {
1918      if (isTableIncluded(tableName) && !tableStates.containsKey(tableName)) {
1919        if (fixMeta) {
1920          MetaTableAccessor.updateTableState(connection, tableName, TableState.State.ENABLED);
1921          TableState newState = MetaTableAccessor.getTableState(connection, tableName);
1922          if (newState == null) {
1923            errors.reportError(ERROR_CODE.NO_TABLE_STATE,
1924                "Unable to change state for table " + tableName + " in meta ");
1925          }
1926        } else {
1927          errors.reportError(ERROR_CODE.NO_TABLE_STATE,
1928              tableName + " has no state in meta ");
1929        }
1930      }
1931    }
1932  }
1933
1934  private void preCheckPermission() throws IOException {
1935    if (shouldIgnorePreCheckPermission()) {
1936      return;
1937    }
1938
1939    Path hbaseDir = CommonFSUtils.getRootDir(getConf());
1940    FileSystem fs = hbaseDir.getFileSystem(getConf());
1941    UserProvider userProvider = UserProvider.instantiate(getConf());
1942    UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1943    FileStatus[] files = fs.listStatus(hbaseDir);
1944    for (FileStatus file : files) {
1945      try {
1946        fs.access(file.getPath(), FsAction.WRITE);
1947      } catch (AccessControlException ace) {
1948        LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1949        errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1950          + " does not have write perms to " + file.getPath()
1951          + ". Please rerun hbck as hdfs user " + file.getOwner());
1952        throw ace;
1953      }
1954    }
1955  }
1956
1957  /**
1958   * Deletes region from meta table
1959   */
1960  private void deleteMetaRegion(HbckRegionInfo hi) throws IOException {
1961    deleteMetaRegion(hi.getMetaEntry().getRegionInfo().getRegionName());
1962  }
1963
1964  /**
1965   * Deletes region from meta table
1966   */
1967  private void deleteMetaRegion(byte[] metaKey) throws IOException {
1968    Delete d = new Delete(metaKey);
1969    meta.delete(d);
1970    LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1971  }
1972
1973  /**
1974   * Reset the split parent region info in meta table
1975   */
1976  private void resetSplitParent(HbckRegionInfo hi) throws IOException {
1977    RowMutations mutations = new RowMutations(hi.getMetaEntry().getRegionInfo().getRegionName());
1978    Delete d = new Delete(hi.getMetaEntry().getRegionInfo().getRegionName());
1979    d.addColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1980    d.addColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1981    mutations.add(d);
1982
1983    RegionInfo hri = RegionInfoBuilder.newBuilder(hi.getMetaEntry().getRegionInfo())
1984      .setOffline(false).setSplit(false).build();
1985    Put p = MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
1986    mutations.add(p);
1987
1988    meta.mutateRow(mutations);
1989    LOG.info("Reset split parent " + hi.getMetaEntry().getRegionInfo().getRegionNameAsString() +
1990      " in META");
1991  }
1992
1993  /**
1994   * This backwards-compatibility wrapper for permanently offlining a region
1995   * that should not be alive.  If the region server does not support the
1996   * "offline" method, it will use the closest unassign method instead.  This
1997   * will basically work until one attempts to disable or delete the affected
1998   * table.  The problem has to do with in-memory only master state, so
1999   * restarting the HMaster or failing over to another should fix this.
2000   */
2001  void offline(byte[] regionName) throws IOException {
2002    String regionString = Bytes.toStringBinary(regionName);
2003    if (!rsSupportsOffline) {
2004      LOG.warn(
2005          "Using unassign region " + regionString + " instead of using offline method, you should" +
2006              " restart HMaster after these repairs");
2007      admin.unassign(regionName, true);
2008      return;
2009    }
2010
2011    // first time we assume the rs's supports #offline.
2012    try {
2013      LOG.info("Offlining region " + regionString);
2014      admin.offline(regionName);
2015    } catch (IOException ioe) {
2016      String notFoundMsg = "java.lang.NoSuchMethodException: " +
2017          "org.apache.hadoop.hbase.master.HMaster.offline([B)";
2018      if (ioe.getMessage().contains(notFoundMsg)) {
2019        LOG.warn("Using unassign region " + regionString +
2020            " instead of using offline method, you should" +
2021            " restart HMaster after these repairs");
2022        rsSupportsOffline = false; // in the future just use unassign
2023        admin.unassign(regionName, true);
2024        return;
2025      }
2026      throw ioe;
2027    }
2028  }
2029
2030  /**
2031   * Attempts to undeploy a region from a region server based in information in
2032   * META.  Any operations that modify the file system should make sure that
2033   * its corresponding region is not deployed to prevent data races.
2034   *
2035   * A separate call is required to update the master in-memory region state
2036   * kept in the AssignementManager.  Because disable uses this state instead of
2037   * that found in META, we can't seem to cleanly disable/delete tables that
2038   * have been hbck fixed.  When used on a version of HBase that does not have
2039   * the offline ipc call exposed on the master (&lt;0.90.5, &lt;0.92.0) a master
2040   * restart or failover may be required.
2041   */
2042  void closeRegion(HbckRegionInfo hi) throws IOException, InterruptedException {
2043    if (hi.getMetaEntry() == null && hi.getHdfsEntry() == null) {
2044      undeployRegions(hi);
2045      return;
2046    }
2047
2048    // get assignment info and hregioninfo from meta.
2049    Get get = new Get(hi.getRegionName());
2050    get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2051    get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
2052    get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
2053    // also get the locations of the replicas to close if the primary region is being closed
2054    if (hi.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
2055      int numReplicas = admin.getDescriptor(hi.getTableName()).getRegionReplication();
2056      for (int i = 0; i < numReplicas; i++) {
2057        get.addColumn(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerColumn(i));
2058        get.addColumn(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getStartCodeColumn(i));
2059      }
2060    }
2061    Result r = meta.get(get);
2062    RegionLocations rl = CatalogFamilyFormat.getRegionLocations(r);
2063    if (rl == null) {
2064      LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
2065          " since meta does not have handle to reach it");
2066      return;
2067    }
2068    for (HRegionLocation h : rl.getRegionLocations()) {
2069      ServerName serverName = h.getServerName();
2070      if (serverName == null) {
2071        errors.reportError("Unable to close region "
2072            + hi.getRegionNameAsString() +  " because meta does not "
2073            + "have handle to reach it.");
2074        continue;
2075      }
2076      RegionInfo hri = h.getRegion();
2077      if (hri == null) {
2078        LOG.warn("Unable to close region " + hi.getRegionNameAsString()
2079            + " because hbase:meta had invalid or missing "
2080            + HConstants.CATALOG_FAMILY_STR + ":"
2081            + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
2082            + " qualifier value.");
2083        continue;
2084      }
2085      // close the region -- close files and remove assignment
2086      HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
2087    }
2088  }
2089
2090  private void undeployRegions(HbckRegionInfo hi) throws IOException, InterruptedException {
2091    undeployRegionsForHbi(hi);
2092    // undeploy replicas of the region (but only if the method is invoked for the primary)
2093    if (hi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
2094      return;
2095    }
2096    int numReplicas = admin.getDescriptor(hi.getTableName()).getRegionReplication();
2097    for (int i = 1; i < numReplicas; i++) {
2098      if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
2099      RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
2100          hi.getPrimaryHRIForDeployedReplica(), i);
2101      HbckRegionInfo h = regionInfoMap.get(hri.getEncodedName());
2102      if (h != null) {
2103        undeployRegionsForHbi(h);
2104        //set skip checks; we undeployed it, and we don't want to evaluate this anymore
2105        //in consistency checks
2106        h.setSkipChecks(true);
2107      }
2108    }
2109  }
2110
2111  private void undeployRegionsForHbi(HbckRegionInfo hi) throws IOException, InterruptedException {
2112    for (HbckRegionInfo.OnlineEntry rse : hi.getOnlineEntries()) {
2113      LOG.debug("Undeploy region "  + rse.getRegionInfo() + " from " + rse.getServerName());
2114      try {
2115        HBaseFsckRepair
2116            .closeRegionSilentlyAndWait(connection, rse.getServerName(), rse.getRegionInfo());
2117        offline(rse.getRegionInfo().getRegionName());
2118      } catch (IOException ioe) {
2119        LOG.warn("Got exception when attempting to offline region "
2120            + Bytes.toString(rse.getRegionInfo().getRegionName()), ioe);
2121      }
2122    }
2123  }
2124
2125  private void tryAssignmentRepair(HbckRegionInfo hbi, String msg) throws IOException,
2126    KeeperException, InterruptedException {
2127    // If we are trying to fix the errors
2128    if (shouldFixAssignments()) {
2129      errors.print(msg);
2130      undeployRegions(hbi);
2131      setShouldRerun();
2132      RegionInfo hri = hbi.getHdfsHRI();
2133      if (hri == null) {
2134        hri = hbi.getMetaEntry().getRegionInfo();
2135      }
2136      HBaseFsckRepair.fixUnassigned(admin, hri);
2137      HBaseFsckRepair.waitUntilAssigned(admin, hri);
2138
2139      // also assign replicas if needed (do it only when this call operates on a primary replica)
2140      if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) return;
2141      int replicationCount = admin.getDescriptor(hri.getTable()).getRegionReplication();
2142      for (int i = 1; i < replicationCount; i++) {
2143        hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
2144        HbckRegionInfo h = regionInfoMap.get(hri.getEncodedName());
2145        if (h != null) {
2146          undeployRegions(h);
2147          //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore
2148          //in consistency checks
2149          h.setSkipChecks(true);
2150        }
2151        HBaseFsckRepair.fixUnassigned(admin, hri);
2152        HBaseFsckRepair.waitUntilAssigned(admin, hri);
2153      }
2154
2155    }
2156  }
2157
2158  /**
2159   * Check a single region for consistency and correct deployment.
2160   */
2161  private void checkRegionConsistency(final String key, final HbckRegionInfo hbi)
2162      throws IOException, KeeperException, InterruptedException {
2163
2164    if (hbi.isSkipChecks()) return;
2165    String descriptiveName = hbi.toString();
2166    boolean inMeta = hbi.getMetaEntry() != null;
2167    // In case not checking HDFS, assume the region is on HDFS
2168    boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
2169    boolean hasMetaAssignment = inMeta && hbi.getMetaEntry().regionServer != null;
2170    boolean isDeployed = !hbi.getDeployedOn().isEmpty();
2171    boolean isMultiplyDeployed = hbi.getDeployedOn().size() > 1;
2172    boolean deploymentMatchesMeta = hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
2173      hbi.getMetaEntry().regionServer.equals(hbi.getDeployedOn().get(0));
2174    boolean splitParent = inMeta && hbi.getMetaEntry().getRegionInfo().isSplit() &&
2175      hbi.getMetaEntry().getRegionInfo().isOffline();
2176    boolean shouldBeDeployed =
2177      inMeta && !isTableDisabled(hbi.getMetaEntry().getRegionInfo().getTable());
2178    boolean recentlyModified = inHdfs &&
2179      hbi.getModTime() + timelag > EnvironmentEdgeManager.currentTime();
2180
2181    // ========== First the healthy cases =============
2182    if (hbi.containsOnlyHdfsEdits()) {
2183      return;
2184    }
2185    if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
2186      return;
2187    } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
2188      LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
2189        "tabled that is not deployed");
2190      return;
2191    } else if (recentlyModified) {
2192      LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
2193      return;
2194    }
2195    // ========== Cases where the region is not in hbase:meta =============
2196    else if (!inMeta && !inHdfs && !isDeployed) {
2197      // We shouldn't have record of this region at all then!
2198      assert false : "Entry for region with no data";
2199    } else if (!inMeta && !inHdfs && isDeployed) {
2200      errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
2201          + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
2202          "deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));
2203      if (shouldFixAssignments()) {
2204        undeployRegions(hbi);
2205      }
2206
2207    } else if (!inMeta && inHdfs && !isDeployed) {
2208      if (hbi.isMerged()) {
2209        // This region has already been merged, the remaining hdfs file will be
2210        // cleaned by CatalogJanitor later
2211        hbi.setSkipChecks(true);
2212        LOG.info("Region " + descriptiveName
2213            + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
2214        return;
2215      }
2216      errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
2217          + descriptiveName + " on HDFS, but not listed in hbase:meta " +
2218          "or deployed on any region server");
2219      // restore region consistency of an adopted orphan
2220      if (shouldFixMeta()) {
2221        if (!hbi.isHdfsRegioninfoPresent()) {
2222          LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
2223              +  " in table integrity repair phase if -fixHdfsOrphans was" +
2224              " used.");
2225          return;
2226        }
2227
2228        RegionInfo hri = hbi.getHdfsHRI();
2229        HbckTableInfo tableInfo = tablesInfo.get(hri.getTable());
2230
2231        for (RegionInfo region : tableInfo.getRegionsFromMeta(this.regionInfoMap)) {
2232          if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0
2233              && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(),
2234                hri.getEndKey()) >= 0)
2235              && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) {
2236            if(region.isSplit() || region.isOffline()) continue;
2237            Path regionDir = hbi.getHdfsRegionDir();
2238            FileSystem fs = regionDir.getFileSystem(getConf());
2239            List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir);
2240            for (Path familyDir : familyDirs) {
2241              List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);
2242              for (Path referenceFilePath : referenceFilePaths) {
2243                Path parentRegionDir =
2244                    StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();
2245                if (parentRegionDir.toString().endsWith(region.getEncodedName())) {
2246                  LOG.warn(hri + " start and stop keys are in the range of " + region
2247                      + ". The region might not be cleaned up from hdfs when region " + region
2248                      + " split failed. Hence deleting from hdfs.");
2249                  HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
2250                    regionDir.getParent(), hri);
2251                  return;
2252                }
2253              }
2254            }
2255          }
2256        }
2257        LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
2258        int numReplicas = admin.getDescriptor(hbi.getTableName()).getRegionReplication();
2259        HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2260            admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
2261              .getLiveServerMetrics().keySet(), numReplicas);
2262
2263        tryAssignmentRepair(hbi, "Trying to reassign region...");
2264      }
2265
2266    } else if (!inMeta && inHdfs && isDeployed) {
2267      errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
2268          + " not in META, but deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));
2269      debugLsr(hbi.getHdfsRegionDir());
2270      if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
2271        // for replicas, this means that we should undeploy the region (we would have
2272        // gone over the primaries and fixed meta holes in first phase under
2273        // checkAndFixConsistency; we shouldn't get the condition !inMeta at
2274        // this stage unless unwanted replica)
2275        if (shouldFixAssignments()) {
2276          undeployRegionsForHbi(hbi);
2277        }
2278      }
2279      if (shouldFixMeta() && hbi.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
2280        if (!hbi.isHdfsRegioninfoPresent()) {
2281          LOG.error("This should have been repaired in table integrity repair phase");
2282          return;
2283        }
2284
2285        LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
2286        int numReplicas = admin.getDescriptor(hbi.getTableName()).getRegionReplication();
2287        HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2288            admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
2289              .getLiveServerMetrics().keySet(), numReplicas);
2290        tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2291      }
2292
2293    // ========== Cases where the region is in hbase:meta =============
2294    } else if (inMeta && inHdfs && !isDeployed && splitParent) {
2295      // check whether this is an actual error, or just transient state where parent
2296      // is not cleaned
2297      if (hbi.getMetaEntry().splitA != null && hbi.getMetaEntry().splitB != null) {
2298        // check that split daughters are there
2299        HbckRegionInfo infoA = this.regionInfoMap.get(hbi.getMetaEntry().splitA.getEncodedName());
2300        HbckRegionInfo infoB = this.regionInfoMap.get(hbi.getMetaEntry().splitB.getEncodedName());
2301        if (infoA != null && infoB != null) {
2302          // we already processed or will process daughters. Move on, nothing to see here.
2303          hbi.setSkipChecks(true);
2304          return;
2305        }
2306      }
2307
2308      // For Replica region, we need to do a similar check. If replica is not split successfully,
2309      // error is going to be reported against primary daughter region.
2310      if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
2311        LOG.info("Region " + descriptiveName + " is a split parent in META, in HDFS, "
2312            + "and not deployed on any region server. This may be transient.");
2313        hbi.setSkipChecks(true);
2314        return;
2315      }
2316
2317      errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2318          + descriptiveName + " is a split parent in META, in HDFS, "
2319          + "and not deployed on any region server. This could be transient, "
2320          + "consider to run the catalog janitor first!");
2321      if (shouldFixSplitParents()) {
2322        setShouldRerun();
2323        resetSplitParent(hbi);
2324      }
2325    } else if (inMeta && !inHdfs && !isDeployed) {
2326      errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2327          + descriptiveName + " found in META, but not in HDFS "
2328          + "or deployed on any region server.");
2329      if (shouldFixMeta()) {
2330        deleteMetaRegion(hbi);
2331      }
2332    } else if (inMeta && !inHdfs && isDeployed) {
2333      errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2334          + " found in META, but not in HDFS, " +
2335          "and deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));
2336      // We treat HDFS as ground truth.  Any information in meta is transient
2337      // and equivalent data can be regenerated.  So, lets unassign and remove
2338      // these problems from META.
2339      if (shouldFixAssignments()) {
2340        errors.print("Trying to fix unassigned region...");
2341        undeployRegions(hbi);
2342      }
2343      if (shouldFixMeta()) {
2344        // wait for it to complete
2345        deleteMetaRegion(hbi);
2346      }
2347    } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2348      errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2349          + " not deployed on any region server.");
2350      tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2351    } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2352      errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2353          "Region " + descriptiveName + " should not be deployed according " +
2354          "to META, but is deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));
2355      if (shouldFixAssignments()) {
2356        errors.print("Trying to close the region " + descriptiveName);
2357        setShouldRerun();
2358        HBaseFsckRepair.fixMultiAssignment(connection, hbi.getMetaEntry().getRegionInfo(),
2359          hbi.getDeployedOn());
2360      }
2361    } else if (inMeta && inHdfs && isMultiplyDeployed) {
2362      errors.reportError(ERROR_CODE.MULTI_DEPLOYED,
2363        "Region " + descriptiveName + " is listed in hbase:meta on region server " +
2364          hbi.getMetaEntry().regionServer + " but is multiply assigned to region servers " +
2365          Joiner.on(", ").join(hbi.getDeployedOn()));
2366      // If we are trying to fix the errors
2367      if (shouldFixAssignments()) {
2368        errors.print("Trying to fix assignment error...");
2369        setShouldRerun();
2370        HBaseFsckRepair.fixMultiAssignment(connection, hbi.getMetaEntry().getRegionInfo(),
2371          hbi.getDeployedOn());
2372      }
2373    } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2374      errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2375          + descriptiveName + " listed in hbase:meta on region server " +
2376          hbi.getMetaEntry().regionServer + " but found on region server " +
2377          hbi.getDeployedOn().get(0));
2378      // If we are trying to fix the errors
2379      if (shouldFixAssignments()) {
2380        errors.print("Trying to fix assignment error...");
2381        setShouldRerun();
2382        HBaseFsckRepair.fixMultiAssignment(connection, hbi.getMetaEntry().getRegionInfo(),
2383          hbi.getDeployedOn());
2384        HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2385      }
2386    } else {
2387      errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2388          " is in an unforeseen state:" +
2389          " inMeta=" + inMeta +
2390          " inHdfs=" + inHdfs +
2391          " isDeployed=" + isDeployed +
2392          " isMultiplyDeployed=" + isMultiplyDeployed +
2393          " deploymentMatchesMeta=" + deploymentMatchesMeta +
2394          " shouldBeDeployed=" + shouldBeDeployed);
2395    }
2396  }
2397
2398  /**
2399   * Checks tables integrity. Goes over all regions and scans the tables.
2400   * Collects all the pieces for each table and checks if there are missing,
2401   * repeated or overlapping ones.
2402   * @throws IOException
2403   */
2404  SortedMap<TableName, HbckTableInfo> checkIntegrity() throws IOException {
2405    tablesInfo = new TreeMap<>();
2406    LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2407    for (HbckRegionInfo hbi : regionInfoMap.values()) {
2408      // Check only valid, working regions
2409      if (hbi.getMetaEntry() == null) {
2410        // this assumes that consistency check has run loadMetaEntry
2411        Path p = hbi.getHdfsRegionDir();
2412        if (p == null) {
2413          errors.report("No regioninfo in Meta or HDFS. " + hbi);
2414        }
2415
2416        // TODO test.
2417        continue;
2418      }
2419      if (hbi.getMetaEntry().regionServer == null) {
2420        errors.detail("Skipping region because no region server: " + hbi);
2421        continue;
2422      }
2423      if (hbi.getMetaEntry().getRegionInfo().isOffline()) {
2424        errors.detail("Skipping region because it is offline: " + hbi);
2425        continue;
2426      }
2427      if (hbi.containsOnlyHdfsEdits()) {
2428        errors.detail("Skipping region because it only contains edits" + hbi);
2429        continue;
2430      }
2431
2432      // Missing regionDir or over-deployment is checked elsewhere. Include
2433      // these cases in modTInfo, so we can evaluate those regions as part of
2434      // the region chain in META
2435      //if (hbi.foundRegionDir == null) continue;
2436      //if (hbi.deployedOn.size() != 1) continue;
2437      if (hbi.getDeployedOn().isEmpty()) {
2438        continue;
2439      }
2440
2441      // We should be safe here
2442      TableName tableName = hbi.getMetaEntry().getRegionInfo().getTable();
2443      HbckTableInfo modTInfo = tablesInfo.get(tableName);
2444      if (modTInfo == null) {
2445        modTInfo = new HbckTableInfo(tableName, this);
2446      }
2447      for (ServerName server : hbi.getDeployedOn()) {
2448        modTInfo.addServer(server);
2449      }
2450
2451      if (!hbi.isSkipChecks()) {
2452        modTInfo.addRegionInfo(hbi);
2453      }
2454
2455      tablesInfo.put(tableName, modTInfo);
2456    }
2457
2458    loadTableInfosForTablesWithNoRegion();
2459
2460    logParallelMerge();
2461    for (HbckTableInfo tInfo : tablesInfo.values()) {
2462      TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2463      if (!tInfo.checkRegionChain(handler)) {
2464        errors.report("Found inconsistency in table " + tInfo.getName());
2465      }
2466    }
2467    return tablesInfo;
2468  }
2469
2470  /** Loads table info's for tables that may not have been included, since there are no
2471   * regions reported for the table, but table dir is there in hdfs
2472   */
2473  private void loadTableInfosForTablesWithNoRegion() throws IOException {
2474    Map<String, TableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2475    for (TableDescriptor htd : allTables.values()) {
2476      if (checkMetaOnly && !htd.isMetaTable()) {
2477        continue;
2478      }
2479
2480      TableName tableName = htd.getTableName();
2481      if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2482        HbckTableInfo tableInfo = new HbckTableInfo(tableName, this);
2483        tableInfo.htds.add(htd);
2484        tablesInfo.put(htd.getTableName(), tableInfo);
2485      }
2486    }
2487  }
2488
2489  /**
2490   * Merge hdfs data by moving from contained HbckRegionInfo into targetRegionDir.
2491   * @return number of file move fixes done to merge regions.
2492   */
2493  public int mergeRegionDirs(Path targetRegionDir, HbckRegionInfo contained) throws IOException {
2494    int fileMoves = 0;
2495    String thread = Thread.currentThread().getName();
2496    LOG.debug("[" + thread + "] Contained region dir after close and pause");
2497    debugLsr(contained.getHdfsRegionDir());
2498
2499    // rename the contained into the container.
2500    FileSystem fs = targetRegionDir.getFileSystem(getConf());
2501    FileStatus[] dirs = null;
2502    try {
2503      dirs = fs.listStatus(contained.getHdfsRegionDir());
2504    } catch (FileNotFoundException fnfe) {
2505      // region we are attempting to merge in is not present!  Since this is a merge, there is
2506      // no harm skipping this region if it does not exist.
2507      if (!fs.exists(contained.getHdfsRegionDir())) {
2508        LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2509            + " is missing. Assuming already sidelined or moved.");
2510      } else {
2511        sidelineRegionDir(fs, contained);
2512      }
2513      return fileMoves;
2514    }
2515
2516    if (dirs == null) {
2517      if (!fs.exists(contained.getHdfsRegionDir())) {
2518        LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2519            + " already sidelined.");
2520      } else {
2521        sidelineRegionDir(fs, contained);
2522      }
2523      return fileMoves;
2524    }
2525
2526    for (FileStatus cf : dirs) {
2527      Path src = cf.getPath();
2528      Path dst =  new Path(targetRegionDir, src.getName());
2529
2530      if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2531        // do not copy the old .regioninfo file.
2532        continue;
2533      }
2534
2535      if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2536        // do not copy the .oldlogs files
2537        continue;
2538      }
2539
2540      LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2541      // FileSystem.rename is inconsistent with directories -- if the
2542      // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2543      // it moves the src into the dst dir resulting in (foo/a/b).  If
2544      // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2545      for (FileStatus hfile : fs.listStatus(src)) {
2546        boolean success = fs.rename(hfile.getPath(), dst);
2547        if (success) {
2548          fileMoves++;
2549        }
2550      }
2551      LOG.debug("[" + thread + "] Sideline directory contents:");
2552      debugLsr(targetRegionDir);
2553    }
2554
2555    // if all success.
2556    sidelineRegionDir(fs, contained);
2557    LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2558        getSidelineDir());
2559    debugLsr(contained.getHdfsRegionDir());
2560
2561    return fileMoves;
2562  }
2563
2564
2565  static class WorkItemOverlapMerge implements Callable<Void> {
2566    private TableIntegrityErrorHandler handler;
2567    Collection<HbckRegionInfo> overlapgroup;
2568
2569    WorkItemOverlapMerge(Collection<HbckRegionInfo> overlapgroup,
2570        TableIntegrityErrorHandler handler) {
2571      this.handler = handler;
2572      this.overlapgroup = overlapgroup;
2573    }
2574
2575    @Override
2576    public Void call() throws Exception {
2577      handler.handleOverlapGroup(overlapgroup);
2578      return null;
2579    }
2580  }
2581
2582  /**
2583   * Return a list of user-space table names whose metadata have not been
2584   * modified in the last few milliseconds specified by timelag
2585   * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
2586   * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
2587   * milliseconds specified by timelag, then the table is a candidate to be returned.
2588   * @return tables that have not been modified recently
2589   * @throws IOException if an error is encountered
2590   */
2591  TableDescriptor[] getTables(AtomicInteger numSkipped) {
2592    List<TableName> tableNames = new ArrayList<>();
2593    long now = EnvironmentEdgeManager.currentTime();
2594
2595    for (HbckRegionInfo hbi : regionInfoMap.values()) {
2596      HbckRegionInfo.MetaEntry info = hbi.getMetaEntry();
2597
2598      // if the start key is zero, then we have found the first region of a table.
2599      // pick only those tables that were not modified in the last few milliseconds.
2600      if (info != null && info.getRegionInfo().getStartKey().length == 0 &&
2601        !info.getRegionInfo().isMetaRegion()) {
2602        if (info.modTime + timelag < now) {
2603          tableNames.add(info.getRegionInfo().getTable());
2604        } else {
2605          numSkipped.incrementAndGet(); // one more in-flux table
2606        }
2607      }
2608    }
2609    return getTableDescriptors(tableNames);
2610  }
2611
2612  TableDescriptor[] getTableDescriptors(List<TableName> tableNames) {
2613      LOG.info("getTableDescriptors == tableNames => " + tableNames);
2614    try (Connection conn = ConnectionFactory.createConnection(getConf());
2615        Admin admin = conn.getAdmin()) {
2616      List<TableDescriptor> tds = admin.listTableDescriptors(tableNames);
2617      return tds.toArray(new TableDescriptor[tds.size()]);
2618    } catch (IOException e) {
2619      LOG.debug("Exception getting table descriptors", e);
2620    }
2621    return new TableDescriptor[0];
2622  }
2623
2624  /**
2625   * Gets the entry in regionInfo corresponding to the the given encoded
2626   * region name. If the region has not been seen yet, a new entry is added
2627   * and returned.
2628   */
2629  private synchronized HbckRegionInfo getOrCreateInfo(String name) {
2630    HbckRegionInfo hbi = regionInfoMap.get(name);
2631    if (hbi == null) {
2632      hbi = new HbckRegionInfo(null);
2633      regionInfoMap.put(name, hbi);
2634    }
2635    return hbi;
2636  }
2637
2638  private void checkAndFixReplication() throws ReplicationException {
2639    ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, errors);
2640    checker.checkUnDeletedQueues();
2641
2642    if (checker.hasUnDeletedQueues() && this.fixReplication) {
2643      checker.fixUnDeletedQueues();
2644      setShouldRerun();
2645    }
2646  }
2647
2648  /**
2649    * Check values in regionInfo for hbase:meta
2650    * Check if zero or more than one regions with hbase:meta are found.
2651    * If there are inconsistencies (i.e. zero or more than one regions
2652    * pretend to be holding the hbase:meta) try to fix that and report an error.
2653    * @throws IOException from HBaseFsckRepair functions
2654    * @throws KeeperException
2655    * @throws InterruptedException
2656    */
2657  boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
2658    Map<Integer, HbckRegionInfo> metaRegions = new HashMap<>();
2659    for (HbckRegionInfo value : regionInfoMap.values()) {
2660      if (value.getMetaEntry() != null && value.getMetaEntry().getRegionInfo().isMetaRegion()) {
2661        metaRegions.put(value.getReplicaId(), value);
2662      }
2663    }
2664    int metaReplication = admin.getDescriptor(TableName.META_TABLE_NAME)
2665        .getRegionReplication();
2666    boolean noProblem = true;
2667    // There will be always entries in regionInfoMap corresponding to hbase:meta & its replicas
2668    // Check the deployed servers. It should be exactly one server for each replica.
2669    for (int i = 0; i < metaReplication; i++) {
2670      HbckRegionInfo metaHbckRegionInfo = metaRegions.remove(i);
2671      List<ServerName> servers = new ArrayList<>();
2672      if (metaHbckRegionInfo != null) {
2673        servers = metaHbckRegionInfo.getDeployedOn();
2674      }
2675      if (servers.size() != 1) {
2676        noProblem = false;
2677        if (servers.isEmpty()) {
2678          assignMetaReplica(i);
2679        } else if (servers.size() > 1) {
2680          errors
2681          .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId " +
2682                       metaHbckRegionInfo.getReplicaId() + " is found on more than one region.");
2683          if (shouldFixAssignments()) {
2684            errors.print("Trying to fix a problem with hbase:meta, replicaId " +
2685                metaHbckRegionInfo.getReplicaId() + "..");
2686            setShouldRerun();
2687            // try fix it (treat is a dupe assignment)
2688            HBaseFsckRepair.fixMultiAssignment(connection,
2689              metaHbckRegionInfo.getMetaEntry().getRegionInfo(), servers);
2690          }
2691        }
2692      }
2693    }
2694    // unassign whatever is remaining in metaRegions. They are excess replicas.
2695    for (Map.Entry<Integer, HbckRegionInfo> entry : metaRegions.entrySet()) {
2696      noProblem = false;
2697      errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2698          "hbase:meta replicas are deployed in excess. Configured " + metaReplication +
2699          ", deployed " + metaRegions.size());
2700      if (shouldFixAssignments()) {
2701        errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() +
2702            " of hbase:meta..");
2703        setShouldRerun();
2704        unassignMetaReplica(entry.getValue());
2705      }
2706    }
2707    // if noProblem is false, rerun hbck with hopefully fixed META
2708    // if noProblem is true, no errors, so continue normally
2709    return noProblem;
2710  }
2711
2712  private void unassignMetaReplica(HbckRegionInfo hi)
2713    throws IOException, InterruptedException, KeeperException {
2714    undeployRegions(hi);
2715    ZKUtil.deleteNode(zkw,
2716      zkw.getZNodePaths().getZNodeForReplica(hi.getMetaEntry().getRegionInfo().getReplicaId()));
2717  }
2718
2719  private void assignMetaReplica(int replicaId)
2720      throws IOException, KeeperException, InterruptedException {
2721    errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " +
2722        replicaId +" is not found on any region.");
2723    if (shouldFixAssignments()) {
2724      errors.print("Trying to fix a problem with hbase:meta..");
2725      setShouldRerun();
2726      // try to fix it (treat it as unassigned region)
2727      RegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
2728          RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId);
2729      HBaseFsckRepair.fixUnassigned(admin, h);
2730      HBaseFsckRepair.waitUntilAssigned(admin, h);
2731    }
2732  }
2733
2734  /**
2735   * Scan hbase:meta, adding all regions found to the regionInfo map.
2736   * @throws IOException if an error is encountered
2737   */
2738  boolean loadMetaEntries() throws IOException {
2739    ClientMetaTableAccessor.Visitor visitor = new ClientMetaTableAccessor.Visitor() {
2740      int countRecord = 1;
2741
2742      // comparator to sort KeyValues with latest modtime
2743      final Comparator<Cell> comp = new Comparator<Cell>() {
2744        @Override
2745        public int compare(Cell k1, Cell k2) {
2746          return Long.compare(k1.getTimestamp(), k2.getTimestamp());
2747        }
2748      };
2749
2750      @Override
2751      public boolean visit(Result result) throws IOException {
2752        try {
2753
2754          // record the latest modification of this META record
2755          long ts =  Collections.max(result.listCells(), comp).getTimestamp();
2756          RegionLocations rl = CatalogFamilyFormat.getRegionLocations(result);
2757          if (rl == null) {
2758            emptyRegionInfoQualifiers.add(result);
2759            errors.reportError(ERROR_CODE.EMPTY_META_CELL,
2760              "Empty REGIONINFO_QUALIFIER found in hbase:meta");
2761            return true;
2762          }
2763          ServerName sn = null;
2764          if (rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID) == null ||
2765              rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID).getRegion() == null) {
2766            emptyRegionInfoQualifiers.add(result);
2767            errors.reportError(ERROR_CODE.EMPTY_META_CELL,
2768              "Empty REGIONINFO_QUALIFIER found in hbase:meta");
2769            return true;
2770          }
2771          RegionInfo hri = rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID).getRegion();
2772          if (!(isTableIncluded(hri.getTable())
2773              || hri.isMetaRegion())) {
2774            return true;
2775          }
2776          PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(result);
2777          for (HRegionLocation h : rl.getRegionLocations()) {
2778            if (h == null || h.getRegion() == null) {
2779              continue;
2780            }
2781            sn = h.getServerName();
2782            hri = h.getRegion();
2783
2784            HbckRegionInfo.MetaEntry m = null;
2785            if (hri.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
2786              m = new HbckRegionInfo.MetaEntry(hri, sn, ts, daughters.getFirst(),
2787                  daughters.getSecond());
2788            } else {
2789              m = new HbckRegionInfo.MetaEntry(hri, sn, ts, null, null);
2790            }
2791            HbckRegionInfo previous = regionInfoMap.get(hri.getEncodedName());
2792            if (previous == null) {
2793              regionInfoMap.put(hri.getEncodedName(), new HbckRegionInfo(m));
2794            } else if (previous.getMetaEntry() == null) {
2795              previous.setMetaEntry(m);
2796            } else {
2797              throw new IOException("Two entries in hbase:meta are same " + previous);
2798            }
2799          }
2800          List<RegionInfo> mergeParents = CatalogFamilyFormat.getMergeRegions(result.rawCells());
2801          if (mergeParents != null) {
2802            for (RegionInfo mergeRegion : mergeParents) {
2803              if (mergeRegion != null) {
2804                // This region is already being merged
2805                HbckRegionInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
2806                hbInfo.setMerged(true);
2807              }
2808            }
2809          }
2810
2811          // show proof of progress to the user, once for every 100 records.
2812          if (countRecord % 100 == 0) {
2813            errors.progress();
2814          }
2815          countRecord++;
2816          return true;
2817        } catch (RuntimeException e) {
2818          LOG.error("Result=" + result);
2819          throw e;
2820        }
2821      }
2822    };
2823    if (!checkMetaOnly) {
2824      // Scan hbase:meta to pick up user regions
2825      MetaTableAccessor.fullScanRegions(connection, visitor);
2826    }
2827
2828    errors.print("");
2829    return true;
2830  }
2831
2832  /**
2833   * Prints summary of all tables found on the system.
2834   */
2835  private void printTableSummary(SortedMap<TableName, HbckTableInfo> tablesInfo) {
2836    StringBuilder sb = new StringBuilder();
2837    int numOfSkippedRegions;
2838    errors.print("Summary:");
2839    for (HbckTableInfo tInfo : tablesInfo.values()) {
2840      numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName())) ?
2841          skippedRegions.get(tInfo.getName()).size() : 0;
2842
2843      if (errors.tableHasErrors(tInfo)) {
2844        errors.print("Table " + tInfo.getName() + " is inconsistent.");
2845      } else if (numOfSkippedRegions > 0){
2846        errors.print("Table " + tInfo.getName() + " is okay (with "
2847          + numOfSkippedRegions + " skipped regions).");
2848      }
2849      else {
2850        errors.print("Table " + tInfo.getName() + " is okay.");
2851      }
2852      errors.print("    Number of regions: " + tInfo.getNumRegions());
2853      if (numOfSkippedRegions > 0) {
2854        Set<String> skippedRegionStrings = skippedRegions.get(tInfo.getName());
2855        System.out.println("    Number of skipped regions: " + numOfSkippedRegions);
2856        System.out.println("      List of skipped regions:");
2857        for(String sr : skippedRegionStrings) {
2858          System.out.println("        " + sr);
2859        }
2860      }
2861      sb.setLength(0); // clear out existing buffer, if any.
2862      sb.append("    Deployed on: ");
2863      for (ServerName server : tInfo.deployedOn) {
2864        sb.append(" " + server.toString());
2865      }
2866      errors.print(sb.toString());
2867    }
2868  }
2869
2870  static HbckErrorReporter getErrorReporter(final Configuration conf)
2871      throws ClassNotFoundException {
2872    Class<? extends HbckErrorReporter> reporter =
2873        conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class,
2874            HbckErrorReporter.class);
2875    return ReflectionUtils.newInstance(reporter, conf);
2876  }
2877
2878  static class PrintingErrorReporter implements HbckErrorReporter {
2879    public int errorCount = 0;
2880    private int showProgress;
2881    // How frequently calls to progress() will create output
2882    private static final int progressThreshold = 100;
2883
2884    Set<HbckTableInfo> errorTables = new HashSet<>();
2885
2886    // for use by unit tests to verify which errors were discovered
2887    private ArrayList<ERROR_CODE> errorList = new ArrayList<>();
2888
2889    @Override
2890    public void clear() {
2891      errorTables.clear();
2892      errorList.clear();
2893      errorCount = 0;
2894    }
2895
2896    @Override
2897    public synchronized void reportError(ERROR_CODE errorCode, String message) {
2898      if (errorCode == ERROR_CODE.WRONG_USAGE) {
2899        System.err.println(message);
2900        return;
2901      }
2902
2903      errorList.add(errorCode);
2904      if (!summary) {
2905        System.out.println("ERROR: " + message);
2906      }
2907      errorCount++;
2908      showProgress = 0;
2909    }
2910
2911    @Override
2912    public synchronized void reportError(ERROR_CODE errorCode, String message,
2913        HbckTableInfo table) {
2914      errorTables.add(table);
2915      reportError(errorCode, message);
2916    }
2917
2918    @Override
2919    public synchronized void reportError(ERROR_CODE errorCode, String message, HbckTableInfo table,
2920                                         HbckRegionInfo info) {
2921      errorTables.add(table);
2922      String reference = "(region " + info.getRegionNameAsString() + ")";
2923      reportError(errorCode, reference + " " + message);
2924    }
2925
2926    @Override
2927    public synchronized void reportError(ERROR_CODE errorCode, String message, HbckTableInfo table,
2928                                         HbckRegionInfo info1, HbckRegionInfo info2) {
2929      errorTables.add(table);
2930      String reference = "(regions " + info1.getRegionNameAsString()
2931          + " and " + info2.getRegionNameAsString() + ")";
2932      reportError(errorCode, reference + " " + message);
2933    }
2934
2935    @Override
2936    public synchronized void reportError(String message) {
2937      reportError(ERROR_CODE.UNKNOWN, message);
2938    }
2939
2940    /**
2941     * Report error information, but do not increment the error count.  Intended for cases
2942     * where the actual error would have been reported previously.
2943     * @param message
2944     */
2945    @Override
2946    public synchronized void report(String message) {
2947      if (! summary) {
2948        System.out.println("ERROR: " + message);
2949      }
2950      showProgress = 0;
2951    }
2952
2953    @Override
2954    public synchronized int summarize() {
2955      System.out.println(Integer.toString(errorCount) +
2956                         " inconsistencies detected.");
2957      if (errorCount == 0) {
2958        System.out.println("Status: OK");
2959        return 0;
2960      } else {
2961        System.out.println("Status: INCONSISTENT");
2962        return -1;
2963      }
2964    }
2965
2966    @Override
2967    public ArrayList<ERROR_CODE> getErrorList() {
2968      return errorList;
2969    }
2970
2971    @Override
2972    public synchronized void print(String message) {
2973      if (!summary) {
2974        System.out.println(message);
2975      }
2976    }
2977
2978    @Override
2979    public boolean tableHasErrors(HbckTableInfo table) {
2980      return errorTables.contains(table);
2981    }
2982
2983    @Override
2984    public void resetErrors() {
2985      errorCount = 0;
2986    }
2987
2988    @Override
2989    public synchronized void detail(String message) {
2990      if (details) {
2991        System.out.println(message);
2992      }
2993      showProgress = 0;
2994    }
2995
2996    @Override
2997    public synchronized void progress() {
2998      if (showProgress++ == progressThreshold) {
2999        if (!summary) {
3000          System.out.print(".");
3001        }
3002        showProgress = 0;
3003      }
3004    }
3005  }
3006
3007  /**
3008   * Contact a region server and get all information from it
3009   */
3010  static class WorkItemRegion implements Callable<Void> {
3011    private final HBaseFsck hbck;
3012    private final ServerName rsinfo;
3013    private final HbckErrorReporter errors;
3014    private final Connection connection;
3015
3016    WorkItemRegion(HBaseFsck hbck, ServerName info, HbckErrorReporter errors,
3017        Connection connection) {
3018      this.hbck = hbck;
3019      this.rsinfo = info;
3020      this.errors = errors;
3021      this.connection = connection;
3022    }
3023
3024    @Override
3025    public synchronized Void call() throws IOException {
3026      errors.progress();
3027      try {
3028        // list all online regions from this region server
3029        List<RegionInfo> regions = connection.getAdmin().getRegions(rsinfo);
3030        regions = filterRegions(regions);
3031
3032        if (details) {
3033          errors.detail(
3034            "RegionServer: " + rsinfo.getServerName() + " number of regions: " + regions.size());
3035          for (RegionInfo rinfo : regions) {
3036            errors.detail("  " + rinfo.getRegionNameAsString() + " id: " + rinfo.getRegionId() +
3037              " encoded_name: " + rinfo.getEncodedName() + " start: " +
3038              Bytes.toStringBinary(rinfo.getStartKey()) + " end: " +
3039              Bytes.toStringBinary(rinfo.getEndKey()));
3040          }
3041        }
3042
3043        // check to see if the existence of this region matches the region in META
3044        for (RegionInfo r : regions) {
3045          HbckRegionInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
3046          hbi.addServer(r, rsinfo);
3047        }
3048      } catch (IOException e) { // unable to connect to the region server.
3049        errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE,
3050          "RegionServer: " + rsinfo.getServerName() + " Unable to fetch region information. " + e);
3051        throw e;
3052      }
3053      return null;
3054    }
3055
3056    private List<RegionInfo> filterRegions(List<RegionInfo> regions) {
3057      List<RegionInfo> ret = Lists.newArrayList();
3058      for (RegionInfo hri : regions) {
3059        if (hri.isMetaRegion() || (!hbck.checkMetaOnly
3060            && hbck.isTableIncluded(hri.getTable()))) {
3061          ret.add(hri);
3062        }
3063      }
3064      return ret;
3065    }
3066  }
3067
3068  /**
3069   * Contact hdfs and get all information about specified table directory into
3070   * regioninfo list.
3071   */
3072  class WorkItemHdfsDir implements Callable<Void> {
3073    private FileStatus tableDir;
3074    private HbckErrorReporter errors;
3075    private FileSystem fs;
3076
3077    WorkItemHdfsDir(FileSystem fs, HbckErrorReporter errors, FileStatus status) {
3078      this.fs = fs;
3079      this.tableDir = status;
3080      this.errors = errors;
3081    }
3082
3083    @Override
3084    public synchronized Void call() throws InterruptedException, ExecutionException {
3085      final Vector<Exception> exceptions = new Vector<>();
3086
3087      try {
3088        final FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
3089        final List<Future<?>> futures = new ArrayList<>(regionDirs.length);
3090
3091        for (final FileStatus regionDir : regionDirs) {
3092          errors.progress();
3093          final String encodedName = regionDir.getPath().getName();
3094          // ignore directories that aren't hexadecimal
3095          if (!encodedName.toLowerCase(Locale.ROOT).matches("[0-9a-f]+")) {
3096            continue;
3097          }
3098
3099          if (!exceptions.isEmpty()) {
3100            break;
3101          }
3102
3103          futures.add(executor.submit(new Runnable() {
3104            @Override
3105            public void run() {
3106              try {
3107                LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
3108
3109                Path regioninfoFile = new Path(regionDir.getPath(), HRegionFileSystem.REGION_INFO_FILE);
3110                boolean regioninfoFileExists = fs.exists(regioninfoFile);
3111
3112                if (!regioninfoFileExists) {
3113                  // As tables become larger it is more and more likely that by the time you
3114                  // reach a given region that it will be gone due to region splits/merges.
3115                  if (!fs.exists(regionDir.getPath())) {
3116                    LOG.warn("By the time we tried to process this region dir it was already gone: "
3117                        + regionDir.getPath());
3118                    return;
3119                  }
3120                }
3121
3122                HbckRegionInfo hbi = HBaseFsck.this.getOrCreateInfo(encodedName);
3123                HbckRegionInfo.HdfsEntry he = new HbckRegionInfo.HdfsEntry();
3124                synchronized (hbi) {
3125                  if (hbi.getHdfsRegionDir() != null) {
3126                    errors.print("Directory " + encodedName + " duplicate??" +
3127                                 hbi.getHdfsRegionDir());
3128                  }
3129
3130                  he.regionDir = regionDir.getPath();
3131                  he.regionDirModTime = regionDir.getModificationTime();
3132                  he.hdfsRegioninfoFilePresent = regioninfoFileExists;
3133                  // we add to orphan list when we attempt to read .regioninfo
3134
3135                  // Set a flag if this region contains only edits
3136                  // This is special case if a region is left after split
3137                  he.hdfsOnlyEdits = true;
3138                  FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
3139                  Path ePath = WALSplitUtil.getRegionDirRecoveredEditsDir(regionDir.getPath());
3140                  for (FileStatus subDir : subDirs) {
3141                    errors.progress();
3142                    String sdName = subDir.getPath().getName();
3143                    if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
3144                      he.hdfsOnlyEdits = false;
3145                      break;
3146                    }
3147                  }
3148                  hbi.setHdfsEntry(he);
3149                }
3150              } catch (Exception e) {
3151                LOG.error("Could not load region dir", e);
3152                exceptions.add(e);
3153              }
3154            }
3155          }));
3156        }
3157
3158        // Ensure all pending tasks are complete (or that we run into an exception)
3159        for (Future<?> f : futures) {
3160          if (!exceptions.isEmpty()) {
3161            break;
3162          }
3163          try {
3164            f.get();
3165          } catch (ExecutionException e) {
3166            LOG.error("Unexpected exec exception!  Should've been caught already.  (Bug?)", e);
3167            // Shouldn't happen, we already logged/caught any exceptions in the Runnable
3168          }
3169        }
3170      } catch (IOException e) {
3171        LOG.error("Cannot execute WorkItemHdfsDir for " + tableDir, e);
3172        exceptions.add(e);
3173      } finally {
3174        if (!exceptions.isEmpty()) {
3175          errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
3176              + tableDir.getPath().getName()
3177              + " Unable to fetch all HDFS region information. ");
3178          // Just throw the first exception as an indication something bad happened
3179          // Don't need to propagate all the exceptions, we already logged them all anyway
3180          throw new ExecutionException("First exception in WorkItemHdfsDir", exceptions.firstElement());
3181        }
3182      }
3183      return null;
3184    }
3185  }
3186
3187  /**
3188   * Contact hdfs and get all information about specified table directory into
3189   * regioninfo list.
3190   */
3191  static class WorkItemHdfsRegionInfo implements Callable<Void> {
3192    private HbckRegionInfo hbi;
3193    private HBaseFsck hbck;
3194    private HbckErrorReporter errors;
3195
3196    WorkItemHdfsRegionInfo(HbckRegionInfo hbi, HBaseFsck hbck, HbckErrorReporter errors) {
3197      this.hbi = hbi;
3198      this.hbck = hbck;
3199      this.errors = errors;
3200    }
3201
3202    @Override
3203    public synchronized Void call() throws IOException {
3204      // only load entries that haven't been loaded yet.
3205      if (hbi.getHdfsHRI() == null) {
3206        try {
3207          errors.progress();
3208          hbi.loadHdfsRegioninfo(hbck.getConf());
3209        } catch (IOException ioe) {
3210          String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
3211              + hbi.getTableName() + " in hdfs dir "
3212              + hbi.getHdfsRegionDir()
3213              + "!  It may be an invalid format or version file.  Treating as "
3214              + "an orphaned regiondir.";
3215          errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
3216          try {
3217            hbck.debugLsr(hbi.getHdfsRegionDir());
3218          } catch (IOException ioe2) {
3219            LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
3220            throw ioe2;
3221          }
3222          hbck.orphanHdfsDirs.add(hbi);
3223          throw ioe;
3224        }
3225      }
3226      return null;
3227    }
3228  }
3229
3230  /**
3231   * Display the full report from fsck. This displays all live and dead region
3232   * servers, and all known regions.
3233   */
3234  public static void setDisplayFullReport() {
3235    details = true;
3236  }
3237
3238  public static boolean shouldDisplayFullReport() {
3239    return details;
3240  }
3241
3242  /**
3243   * Set exclusive mode.
3244   */
3245  public static void setForceExclusive() {
3246    forceExclusive = true;
3247  }
3248
3249  /**
3250   * Only one instance of hbck can modify HBase at a time.
3251   */
3252  public boolean isExclusive() {
3253    return fixAny || forceExclusive;
3254  }
3255
3256  /**
3257   * Set summary mode.
3258   * Print only summary of the tables and status (OK or INCONSISTENT)
3259   */
3260  static void setSummary() {
3261    summary = true;
3262  }
3263
3264  /**
3265   * Set hbase:meta check mode.
3266   * Print only info about hbase:meta table deployment/state
3267   */
3268  void setCheckMetaOnly() {
3269    checkMetaOnly = true;
3270  }
3271
3272  /**
3273   * Set region boundaries check mode.
3274   */
3275  void setRegionBoundariesCheck() {
3276    checkRegionBoundaries = true;
3277  }
3278
3279  /**
3280   * Set replication fix mode.
3281   */
3282  public void setFixReplication(boolean shouldFix) {
3283    fixReplication = shouldFix;
3284    fixAny |= shouldFix;
3285  }
3286
3287  public void setCleanReplicationBarrier(boolean shouldClean) {
3288    cleanReplicationBarrier = shouldClean;
3289  }
3290
3291  /**
3292   * Check if we should rerun fsck again. This checks if we've tried to
3293   * fix something and we should rerun fsck tool again.
3294   * Display the full report from fsck. This displays all live and dead
3295   * region servers, and all known regions.
3296   */
3297  void setShouldRerun() {
3298    rerun = true;
3299  }
3300
3301  public boolean shouldRerun() {
3302    return rerun;
3303  }
3304
3305  /**
3306   * Fix inconsistencies found by fsck. This should try to fix errors (if any)
3307   * found by fsck utility.
3308   */
3309  public void setFixAssignments(boolean shouldFix) {
3310    fixAssignments = shouldFix;
3311    fixAny |= shouldFix;
3312  }
3313
3314  boolean shouldFixAssignments() {
3315    return fixAssignments;
3316  }
3317
3318  public void setFixMeta(boolean shouldFix) {
3319    fixMeta = shouldFix;
3320    fixAny |= shouldFix;
3321  }
3322
3323  boolean shouldFixMeta() {
3324    return fixMeta;
3325  }
3326
3327  public void setFixEmptyMetaCells(boolean shouldFix) {
3328    fixEmptyMetaCells = shouldFix;
3329    fixAny |= shouldFix;
3330  }
3331
3332  boolean shouldFixEmptyMetaCells() {
3333    return fixEmptyMetaCells;
3334  }
3335
3336  public void setCheckHdfs(boolean checking) {
3337    checkHdfs = checking;
3338  }
3339
3340  boolean shouldCheckHdfs() {
3341    return checkHdfs;
3342  }
3343
3344  public void setFixHdfsHoles(boolean shouldFix) {
3345    fixHdfsHoles = shouldFix;
3346    fixAny |= shouldFix;
3347  }
3348
3349  boolean shouldFixHdfsHoles() {
3350    return fixHdfsHoles;
3351  }
3352
3353  public void setFixTableOrphans(boolean shouldFix) {
3354    fixTableOrphans = shouldFix;
3355    fixAny |= shouldFix;
3356  }
3357
3358  boolean shouldFixTableOrphans() {
3359    return fixTableOrphans;
3360  }
3361
3362  public void setFixHdfsOverlaps(boolean shouldFix) {
3363    fixHdfsOverlaps = shouldFix;
3364    fixAny |= shouldFix;
3365  }
3366
3367  boolean shouldFixHdfsOverlaps() {
3368    return fixHdfsOverlaps;
3369  }
3370
3371  public void setFixHdfsOrphans(boolean shouldFix) {
3372    fixHdfsOrphans = shouldFix;
3373    fixAny |= shouldFix;
3374  }
3375
3376  boolean shouldFixHdfsOrphans() {
3377    return fixHdfsOrphans;
3378  }
3379
3380  public void setFixVersionFile(boolean shouldFix) {
3381    fixVersionFile = shouldFix;
3382    fixAny |= shouldFix;
3383  }
3384
3385  public boolean shouldFixVersionFile() {
3386    return fixVersionFile;
3387  }
3388
3389  public void setSidelineBigOverlaps(boolean sbo) {
3390    this.sidelineBigOverlaps = sbo;
3391  }
3392
3393  public boolean shouldSidelineBigOverlaps() {
3394    return sidelineBigOverlaps;
3395  }
3396
3397  public void setFixSplitParents(boolean shouldFix) {
3398    fixSplitParents = shouldFix;
3399    fixAny |= shouldFix;
3400  }
3401
3402  public void setRemoveParents(boolean shouldFix) {
3403    removeParents = shouldFix;
3404    fixAny |= shouldFix;
3405  }
3406
3407  boolean shouldFixSplitParents() {
3408    return fixSplitParents;
3409  }
3410
3411  boolean shouldRemoveParents() {
3412    return removeParents;
3413  }
3414
3415  public void setFixReferenceFiles(boolean shouldFix) {
3416    fixReferenceFiles = shouldFix;
3417    fixAny |= shouldFix;
3418  }
3419
3420  boolean shouldFixReferenceFiles() {
3421    return fixReferenceFiles;
3422  }
3423
3424  public void setFixHFileLinks(boolean shouldFix) {
3425    fixHFileLinks = shouldFix;
3426    fixAny |= shouldFix;
3427  }
3428
3429  boolean shouldFixHFileLinks() {
3430    return fixHFileLinks;
3431  }
3432
3433  public boolean shouldIgnorePreCheckPermission() {
3434    return !fixAny || ignorePreCheckPermission;
3435  }
3436
3437  public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
3438    this.ignorePreCheckPermission = ignorePreCheckPermission;
3439  }
3440
3441  /**
3442   * @param mm maximum number of regions to merge into a single region.
3443   */
3444  public void setMaxMerge(int mm) {
3445    this.maxMerge = mm;
3446  }
3447
3448  public int getMaxMerge() {
3449    return maxMerge;
3450  }
3451
3452  public void setMaxOverlapsToSideline(int mo) {
3453    this.maxOverlapsToSideline = mo;
3454  }
3455
3456  public int getMaxOverlapsToSideline() {
3457    return maxOverlapsToSideline;
3458  }
3459
3460  /**
3461   * Only check/fix tables specified by the list,
3462   * Empty list means all tables are included.
3463   */
3464  boolean isTableIncluded(TableName table) {
3465    return (tablesIncluded.isEmpty()) || tablesIncluded.contains(table);
3466  }
3467
3468  public void includeTable(TableName table) {
3469    tablesIncluded.add(table);
3470  }
3471
3472  Set<TableName> getIncludedTables() {
3473    return new HashSet<>(tablesIncluded);
3474  }
3475
3476  /**
3477   * We are interested in only those tables that have not changed their state in
3478   * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
3479   * @param seconds - the time in seconds
3480   */
3481  public void setTimeLag(long seconds) {
3482    timelag = seconds * 1000; // convert to milliseconds
3483  }
3484
3485  /**
3486   *
3487   * @param sidelineDir - HDFS path to sideline data
3488   */
3489  public void setSidelineDir(String sidelineDir) {
3490    this.sidelineDir = new Path(sidelineDir);
3491  }
3492
3493  protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
3494    return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
3495  }
3496
3497  public HFileCorruptionChecker getHFilecorruptionChecker() {
3498    return hfcc;
3499  }
3500
3501  public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
3502    this.hfcc = hfcc;
3503  }
3504
3505  public void setRetCode(int code) {
3506    this.retcode = code;
3507  }
3508
3509  public int getRetCode() {
3510    return retcode;
3511  }
3512
3513  protected HBaseFsck printUsageAndExit() {
3514    StringWriter sw = new StringWriter(2048);
3515    PrintWriter out = new PrintWriter(sw);
3516    out.println("");
3517    out.println("-----------------------------------------------------------------------");
3518    out.println("NOTE: As of HBase version 2.0, the hbck tool is significantly changed.");
3519    out.println("In general, all Read-Only options are supported and can be be used");
3520    out.println("safely. Most -fix/ -repair options are NOT supported. Please see usage");
3521    out.println("below for details on which options are not supported.");
3522    out.println("-----------------------------------------------------------------------");
3523    out.println("");
3524    out.println("Usage: fsck [opts] {only tables}");
3525    out.println(" where [opts] are:");
3526    out.println("   -help Display help options (this)");
3527    out.println("   -details Display full report of all regions.");
3528    out.println("   -timelag <timeInSeconds>  Process only regions that " +
3529                       " have not experienced any metadata updates in the last " +
3530                       " <timeInSeconds> seconds.");
3531    out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
3532        " before checking if the fix worked if run with -fix");
3533    out.println("   -summary Print only summary of the tables and status.");
3534    out.println("   -metaonly Only check the state of the hbase:meta table.");
3535    out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
3536    out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
3537    out.println("   -exclusive Abort if another hbck is exclusive or fixing.");
3538
3539    out.println("");
3540    out.println("  Datafile Repair options: (expert features, use with caution!)");
3541    out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
3542    out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
3543
3544    out.println("");
3545    out.println(" Replication options");
3546    out.println("   -fixReplication   Deletes replication queues for removed peers");
3547
3548    out.println("");
3549    out.println("  Metadata Repair options supported as of version 2.0: (expert features, use with caution!)");
3550    out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
3551    out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
3552    out.println("   -fixHFileLinks  Try to offline lingering HFileLinks");
3553    out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
3554        + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
3555    out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
3556
3557    out.println("");
3558    out.println("NOTE: Following options are NOT supported as of HBase version 2.0+.");
3559    out.println("");
3560    out.println("  UNSUPPORTED Metadata Repair options: (expert features, use with caution!)");
3561    out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
3562    out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
3563    out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
3564    out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
3565    out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
3566    out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
3567    out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
3568    out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
3569    out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
3570    out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
3571    out.println("   -fixSplitParents  Try to force offline split parents to be online.");
3572    out.println("   -removeParents    Try to offline and sideline lingering parents and keep daughter regions.");
3573    out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
3574        + " (empty REGIONINFO_QUALIFIER rows)");
3575
3576    out.println("");
3577    out.println("  UNSUPPORTED Metadata Repair shortcuts");
3578    out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
3579        "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles" +
3580        "-fixHFileLinks");
3581    out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
3582    out.println("");
3583    out.println(" Replication options");
3584    out.println("   -fixReplication   Deletes replication queues for removed peers");
3585    out.println("   -cleanReplicationBrarier [tableName] clean the replication barriers " +
3586        "of a specified table, tableName is required");
3587    out.flush();
3588    errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
3589
3590    setRetCode(-2);
3591    return this;
3592  }
3593
3594  /**
3595   * Main program
3596   *
3597   * @param args
3598   * @throws Exception
3599   */
3600  public static void main(String[] args) throws Exception {
3601    // create a fsck object
3602    Configuration conf = HBaseConfiguration.create();
3603    Path hbasedir = CommonFSUtils.getRootDir(conf);
3604    URI defaultFs = hbasedir.getFileSystem(conf).getUri();
3605    CommonFSUtils.setFsDefault(conf, new Path(defaultFs));
3606    int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
3607    System.exit(ret);
3608  }
3609
3610  /**
3611   * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
3612   */
3613  static class HBaseFsckTool extends Configured implements Tool {
3614    HBaseFsckTool(Configuration conf) { super(conf); }
3615    @Override
3616    public int run(String[] args) throws Exception {
3617      HBaseFsck hbck = new HBaseFsck(getConf());
3618      hbck.exec(hbck.executor, args);
3619      hbck.close();
3620      return hbck.getRetCode();
3621    }
3622  }
3623
3624  public HBaseFsck exec(ExecutorService exec, String[] args)
3625      throws KeeperException, IOException, InterruptedException, ReplicationException {
3626    long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
3627
3628    boolean checkCorruptHFiles = false;
3629    boolean sidelineCorruptHFiles = false;
3630
3631    // Process command-line args.
3632    for (int i = 0; i < args.length; i++) {
3633      String cmd = args[i];
3634      if (cmd.equals("-help") || cmd.equals("-h")) {
3635        return printUsageAndExit();
3636      } else if (cmd.equals("-details")) {
3637        setDisplayFullReport();
3638      } else if (cmd.equals("-exclusive")) {
3639        setForceExclusive();
3640      } else if (cmd.equals("-timelag")) {
3641        if (i == args.length - 1) {
3642          errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
3643          return printUsageAndExit();
3644        }
3645        try {
3646          long timelag = Long.parseLong(args[++i]);
3647          setTimeLag(timelag);
3648        } catch (NumberFormatException e) {
3649          errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
3650          return printUsageAndExit();
3651        }
3652      } else if (cmd.equals("-sleepBeforeRerun")) {
3653        if (i == args.length - 1) {
3654          errors.reportError(ERROR_CODE.WRONG_USAGE,
3655            "HBaseFsck: -sleepBeforeRerun needs a value.");
3656          return printUsageAndExit();
3657        }
3658        try {
3659          sleepBeforeRerun = Long.parseLong(args[++i]);
3660        } catch (NumberFormatException e) {
3661          errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
3662          return printUsageAndExit();
3663        }
3664      } else if (cmd.equals("-sidelineDir")) {
3665        if (i == args.length - 1) {
3666          errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
3667          return printUsageAndExit();
3668        }
3669        setSidelineDir(args[++i]);
3670      } else if (cmd.equals("-fix")) {
3671        errors.reportError(ERROR_CODE.WRONG_USAGE,
3672          "This option is deprecated, please use  -fixAssignments instead.");
3673        setFixAssignments(true);
3674      } else if (cmd.equals("-fixAssignments")) {
3675        setFixAssignments(true);
3676      } else if (cmd.equals("-fixMeta")) {
3677        setFixMeta(true);
3678      } else if (cmd.equals("-noHdfsChecking")) {
3679        setCheckHdfs(false);
3680      } else if (cmd.equals("-fixHdfsHoles")) {
3681        setFixHdfsHoles(true);
3682      } else if (cmd.equals("-fixHdfsOrphans")) {
3683        setFixHdfsOrphans(true);
3684      } else if (cmd.equals("-fixTableOrphans")) {
3685        setFixTableOrphans(true);
3686      } else if (cmd.equals("-fixHdfsOverlaps")) {
3687        setFixHdfsOverlaps(true);
3688      } else if (cmd.equals("-fixVersionFile")) {
3689        setFixVersionFile(true);
3690      } else if (cmd.equals("-sidelineBigOverlaps")) {
3691        setSidelineBigOverlaps(true);
3692      } else if (cmd.equals("-fixSplitParents")) {
3693        setFixSplitParents(true);
3694      } else if (cmd.equals("-removeParents")) {
3695        setRemoveParents(true);
3696      } else if (cmd.equals("-ignorePreCheckPermission")) {
3697        setIgnorePreCheckPermission(true);
3698      } else if (cmd.equals("-checkCorruptHFiles")) {
3699        checkCorruptHFiles = true;
3700      } else if (cmd.equals("-sidelineCorruptHFiles")) {
3701        sidelineCorruptHFiles = true;
3702      } else if (cmd.equals("-fixReferenceFiles")) {
3703        setFixReferenceFiles(true);
3704      } else if (cmd.equals("-fixHFileLinks")) {
3705        setFixHFileLinks(true);
3706      } else if (cmd.equals("-fixEmptyMetaCells")) {
3707        setFixEmptyMetaCells(true);
3708      } else if (cmd.equals("-repair")) {
3709        // this attempts to merge overlapping hdfs regions, needs testing
3710        // under load
3711        setFixHdfsHoles(true);
3712        setFixHdfsOrphans(true);
3713        setFixMeta(true);
3714        setFixAssignments(true);
3715        setFixHdfsOverlaps(true);
3716        setFixVersionFile(true);
3717        setSidelineBigOverlaps(true);
3718        setFixSplitParents(false);
3719        setCheckHdfs(true);
3720        setFixReferenceFiles(true);
3721        setFixHFileLinks(true);
3722      } else if (cmd.equals("-repairHoles")) {
3723        // this will make all missing hdfs regions available but may lose data
3724        setFixHdfsHoles(true);
3725        setFixHdfsOrphans(false);
3726        setFixMeta(true);
3727        setFixAssignments(true);
3728        setFixHdfsOverlaps(false);
3729        setSidelineBigOverlaps(false);
3730        setFixSplitParents(false);
3731        setCheckHdfs(true);
3732      } else if (cmd.equals("-maxOverlapsToSideline")) {
3733        if (i == args.length - 1) {
3734          errors.reportError(ERROR_CODE.WRONG_USAGE,
3735            "-maxOverlapsToSideline needs a numeric value argument.");
3736          return printUsageAndExit();
3737        }
3738        try {
3739          int maxOverlapsToSideline = Integer.parseInt(args[++i]);
3740          setMaxOverlapsToSideline(maxOverlapsToSideline);
3741        } catch (NumberFormatException e) {
3742          errors.reportError(ERROR_CODE.WRONG_USAGE,
3743            "-maxOverlapsToSideline needs a numeric value argument.");
3744          return printUsageAndExit();
3745        }
3746      } else if (cmd.equals("-maxMerge")) {
3747        if (i == args.length - 1) {
3748          errors.reportError(ERROR_CODE.WRONG_USAGE,
3749            "-maxMerge needs a numeric value argument.");
3750          return printUsageAndExit();
3751        }
3752        try {
3753          int maxMerge = Integer.parseInt(args[++i]);
3754          setMaxMerge(maxMerge);
3755        } catch (NumberFormatException e) {
3756          errors.reportError(ERROR_CODE.WRONG_USAGE,
3757            "-maxMerge needs a numeric value argument.");
3758          return printUsageAndExit();
3759        }
3760      } else if (cmd.equals("-summary")) {
3761        setSummary();
3762      } else if (cmd.equals("-metaonly")) {
3763        setCheckMetaOnly();
3764      } else if (cmd.equals("-boundaries")) {
3765        setRegionBoundariesCheck();
3766      } else if (cmd.equals("-fixReplication")) {
3767        setFixReplication(true);
3768      } else if (cmd.equals("-cleanReplicationBarrier")) {
3769        setCleanReplicationBarrier(true);
3770        if(args[++i].startsWith("-")){
3771          printUsageAndExit();
3772        }
3773        setCleanReplicationBarrierTable(args[i]);
3774      } else if (cmd.startsWith("-")) {
3775        errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
3776        return printUsageAndExit();
3777      } else {
3778        includeTable(TableName.valueOf(cmd));
3779        errors.print("Allow checking/fixes for table: " + cmd);
3780      }
3781    }
3782
3783    errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
3784
3785    // pre-check current user has FS write permission or not
3786    try {
3787      preCheckPermission();
3788    } catch (IOException ioe) {
3789      Runtime.getRuntime().exit(-1);
3790    }
3791
3792    // do the real work of hbck
3793    connect();
3794
3795    // after connecting to server above, we have server version
3796    // check if unsupported option is specified based on server version
3797    if (!isOptionsSupported(args)) {
3798      return printUsageAndExit();
3799    }
3800
3801    try {
3802      // if corrupt file mode is on, first fix them since they may be opened later
3803      if (checkCorruptHFiles || sidelineCorruptHFiles) {
3804        LOG.info("Checking all hfiles for corruption");
3805        HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
3806        setHFileCorruptionChecker(hfcc); // so we can get result
3807        Collection<TableName> tables = getIncludedTables();
3808        Collection<Path> tableDirs = new ArrayList<>();
3809        Path rootdir = CommonFSUtils.getRootDir(getConf());
3810        if (tables.size() > 0) {
3811          for (TableName t : tables) {
3812            tableDirs.add(CommonFSUtils.getTableDir(rootdir, t));
3813          }
3814        } else {
3815          tableDirs = FSUtils.getTableDirs(CommonFSUtils.getCurrentFileSystem(getConf()), rootdir);
3816        }
3817        hfcc.checkTables(tableDirs);
3818        hfcc.report(errors);
3819      }
3820
3821      // check and fix table integrity, region consistency.
3822      int code = onlineHbck();
3823      setRetCode(code);
3824      // If we have changed the HBase state it is better to run hbck again
3825      // to see if we haven't broken something else in the process.
3826      // We run it only once more because otherwise we can easily fall into
3827      // an infinite loop.
3828      if (shouldRerun()) {
3829        try {
3830          LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
3831          Thread.sleep(sleepBeforeRerun);
3832        } catch (InterruptedException ie) {
3833          LOG.warn("Interrupted while sleeping");
3834          return this;
3835        }
3836        // Just report
3837        setFixAssignments(false);
3838        setFixMeta(false);
3839        setFixHdfsHoles(false);
3840        setFixHdfsOverlaps(false);
3841        setFixVersionFile(false);
3842        setFixTableOrphans(false);
3843        errors.resetErrors();
3844        code = onlineHbck();
3845        setRetCode(code);
3846      }
3847    } finally {
3848      IOUtils.closeQuietly(this);
3849    }
3850    return this;
3851  }
3852
3853  private boolean isOptionsSupported(String[] args) {
3854    boolean result = true;
3855    String hbaseServerVersion = status.getHBaseVersion();
3856    if (VersionInfo.compareVersion("2.any.any", hbaseServerVersion) < 0) {
3857      // Process command-line args.
3858      for (String arg : args) {
3859        if (unsupportedOptionsInV2.contains(arg)) {
3860          errors.reportError(ERROR_CODE.UNSUPPORTED_OPTION,
3861              "option '" + arg + "' is not " + "supportted!");
3862          result = false;
3863          break;
3864        }
3865      }
3866    }
3867    return result;
3868  }
3869
3870  public void setCleanReplicationBarrierTable(String cleanReplicationBarrierTable) {
3871    this.cleanReplicationBarrierTable = TableName.valueOf(cleanReplicationBarrierTable);
3872  }
3873
3874  public void cleanReplicationBarrier() throws IOException {
3875    if (!cleanReplicationBarrier || cleanReplicationBarrierTable == null) {
3876      return;
3877    }
3878    if (cleanReplicationBarrierTable.isSystemTable()) {
3879      errors.reportError(ERROR_CODE.INVALID_TABLE,
3880        "invalid table: " + cleanReplicationBarrierTable);
3881      return;
3882    }
3883
3884    boolean isGlobalScope = false;
3885    try {
3886      isGlobalScope = admin.getDescriptor(cleanReplicationBarrierTable).hasGlobalReplicationScope();
3887    } catch (TableNotFoundException e) {
3888      LOG.info("we may need to clean some erroneous data due to bugs");
3889    }
3890
3891    if (isGlobalScope) {
3892      errors.reportError(ERROR_CODE.INVALID_TABLE,
3893        "table's replication scope is global: " + cleanReplicationBarrierTable);
3894      return;
3895    }
3896    List<byte[]> regionNames = new ArrayList<>();
3897    Scan barrierScan = new Scan();
3898    barrierScan.setCaching(100);
3899    barrierScan.addFamily(HConstants.REPLICATION_BARRIER_FAMILY);
3900    barrierScan
3901        .withStartRow(ClientMetaTableAccessor.getTableStartRowForMeta(cleanReplicationBarrierTable,
3902          ClientMetaTableAccessor.QueryType.REGION))
3903        .withStopRow(ClientMetaTableAccessor.getTableStopRowForMeta(cleanReplicationBarrierTable,
3904          ClientMetaTableAccessor.QueryType.REGION));
3905    Result result;
3906    try (ResultScanner scanner = meta.getScanner(barrierScan)) {
3907      while ((result = scanner.next()) != null) {
3908        regionNames.add(result.getRow());
3909      }
3910    }
3911    if (regionNames.size() <= 0) {
3912      errors.reportError(ERROR_CODE.INVALID_TABLE,
3913        "there is no barriers of this table: " + cleanReplicationBarrierTable);
3914      return;
3915    }
3916    ReplicationQueueStorage queueStorage =
3917        ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
3918    List<ReplicationPeerDescription> peerDescriptions = admin.listReplicationPeers();
3919    if (peerDescriptions != null && peerDescriptions.size() > 0) {
3920      List<String> peers = peerDescriptions.stream()
3921          .filter(peerConfig -> peerConfig.getPeerConfig()
3922            .needToReplicate(cleanReplicationBarrierTable))
3923          .map(peerConfig -> peerConfig.getPeerId()).collect(Collectors.toList());
3924      try {
3925        List<String> batch = new ArrayList<>();
3926        for (String peer : peers) {
3927          for (byte[] regionName : regionNames) {
3928            batch.add(RegionInfo.encodeRegionName(regionName));
3929            if (batch.size() % 100 == 0) {
3930              queueStorage.removeLastSequenceIds(peer, batch);
3931              batch.clear();
3932            }
3933          }
3934          if (batch.size() > 0) {
3935            queueStorage.removeLastSequenceIds(peer, batch);
3936            batch.clear();
3937          }
3938        }
3939      } catch (ReplicationException re) {
3940        throw new IOException(re);
3941      }
3942    }
3943    for (byte[] regionName : regionNames) {
3944      meta.delete(new Delete(regionName).addFamily(HConstants.REPLICATION_BARRIER_FAMILY));
3945    }
3946    setShouldRerun();
3947  }
3948
3949  /**
3950   * ls -r for debugging purposes
3951   */
3952  void debugLsr(Path p) throws IOException {
3953    debugLsr(getConf(), p, errors);
3954  }
3955
3956  /**
3957   * ls -r for debugging purposes
3958   */
3959  public static void debugLsr(Configuration conf,
3960      Path p) throws IOException {
3961    debugLsr(conf, p, new PrintingErrorReporter());
3962  }
3963
3964  /**
3965   * ls -r for debugging purposes
3966   */
3967  public static void debugLsr(Configuration conf,
3968      Path p, HbckErrorReporter errors) throws IOException {
3969    if (!LOG.isDebugEnabled() || p == null) {
3970      return;
3971    }
3972    FileSystem fs = p.getFileSystem(conf);
3973
3974    if (!fs.exists(p)) {
3975      // nothing
3976      return;
3977    }
3978    errors.print(p.toString());
3979
3980    if (fs.isFile(p)) {
3981      return;
3982    }
3983
3984    if (fs.getFileStatus(p).isDirectory()) {
3985      FileStatus[] fss= fs.listStatus(p);
3986      for (FileStatus status : fss) {
3987        debugLsr(conf, status.getPath(), errors);
3988      }
3989    }
3990  }
3991}