001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import java.io.Closeable;
021import java.io.FileNotFoundException;
022import java.io.IOException;
023import java.io.InterruptedIOException;
024import java.io.PrintWriter;
025import java.io.StringWriter;
026import java.net.InetAddress;
027import java.net.URI;
028import java.util.ArrayList;
029import java.util.Collection;
030import java.util.Collections;
031import java.util.Comparator;
032import java.util.EnumSet;
033import java.util.HashMap;
034import java.util.HashSet;
035import java.util.Iterator;
036import java.util.List;
037import java.util.Locale;
038import java.util.Map;
039import java.util.Map.Entry;
040import java.util.Objects;
041import java.util.Optional;
042import java.util.Set;
043import java.util.SortedMap;
044import java.util.TreeMap;
045import java.util.Vector;
046import java.util.concurrent.Callable;
047import java.util.concurrent.ConcurrentSkipListMap;
048import java.util.concurrent.ExecutionException;
049import java.util.concurrent.ExecutorService;
050import java.util.concurrent.Executors;
051import java.util.concurrent.Future;
052import java.util.concurrent.FutureTask;
053import java.util.concurrent.ScheduledThreadPoolExecutor;
054import java.util.concurrent.TimeUnit;
055import java.util.concurrent.TimeoutException;
056import java.util.concurrent.atomic.AtomicBoolean;
057import java.util.concurrent.atomic.AtomicInteger;
058import java.util.stream.Collectors;
059
060import org.apache.commons.io.IOUtils;
061import org.apache.commons.lang3.StringUtils;
062import org.apache.hadoop.conf.Configuration;
063import org.apache.hadoop.conf.Configured;
064import org.apache.hadoop.fs.FSDataOutputStream;
065import org.apache.hadoop.fs.FileStatus;
066import org.apache.hadoop.fs.FileSystem;
067import org.apache.hadoop.fs.Path;
068import org.apache.hadoop.fs.permission.FsAction;
069import org.apache.hadoop.fs.permission.FsPermission;
070import org.apache.hadoop.hbase.Abortable;
071import org.apache.hadoop.hbase.Cell;
072import org.apache.hadoop.hbase.CellUtil;
073import org.apache.hadoop.hbase.ClusterMetrics;
074import org.apache.hadoop.hbase.ClusterMetrics.Option;
075import org.apache.hadoop.hbase.HBaseConfiguration;
076import org.apache.hadoop.hbase.HBaseInterfaceAudience;
077import org.apache.hadoop.hbase.HConstants;
078import org.apache.hadoop.hbase.HRegionLocation;
079import org.apache.hadoop.hbase.KeyValue;
080import org.apache.hadoop.hbase.MasterNotRunningException;
081import org.apache.hadoop.hbase.MetaTableAccessor;
082import org.apache.hadoop.hbase.RegionLocations;
083import org.apache.hadoop.hbase.ServerName;
084import org.apache.hadoop.hbase.TableName;
085import org.apache.hadoop.hbase.TableNotFoundException;
086import org.apache.hadoop.hbase.ZooKeeperConnectionException;
087import org.apache.hadoop.hbase.client.Admin;
088import org.apache.hadoop.hbase.client.ClusterConnection;
089import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
090import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
091import org.apache.hadoop.hbase.client.Connection;
092import org.apache.hadoop.hbase.client.ConnectionFactory;
093import org.apache.hadoop.hbase.client.Delete;
094import org.apache.hadoop.hbase.client.Get;
095import org.apache.hadoop.hbase.client.Put;
096import org.apache.hadoop.hbase.client.RegionInfo;
097import org.apache.hadoop.hbase.client.RegionInfoBuilder;
098import org.apache.hadoop.hbase.client.RegionReplicaUtil;
099import org.apache.hadoop.hbase.client.Result;
100import org.apache.hadoop.hbase.client.ResultScanner;
101import org.apache.hadoop.hbase.client.RowMutations;
102import org.apache.hadoop.hbase.client.Scan;
103import org.apache.hadoop.hbase.client.Table;
104import org.apache.hadoop.hbase.client.TableDescriptor;
105import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
106import org.apache.hadoop.hbase.client.TableState;
107import org.apache.hadoop.hbase.io.FileLink;
108import org.apache.hadoop.hbase.io.HFileLink;
109import org.apache.hadoop.hbase.io.hfile.CacheConfig;
110import org.apache.hadoop.hbase.io.hfile.HFile;
111import org.apache.hadoop.hbase.master.RegionState;
112import org.apache.hadoop.hbase.regionserver.HRegion;
113import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
114import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
115import org.apache.hadoop.hbase.replication.ReplicationException;
116import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
117import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
118import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
119import org.apache.hadoop.hbase.security.AccessDeniedException;
120import org.apache.hadoop.hbase.security.UserProvider;
121import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
122import org.apache.hadoop.hbase.util.HbckErrorReporter.ERROR_CODE;
123import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
124import org.apache.hadoop.hbase.util.hbck.ReplicationChecker;
125import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
126import org.apache.hadoop.hbase.wal.WALSplitter;
127import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
128import org.apache.hadoop.hbase.zookeeper.ZKUtil;
129import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
130import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
131import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
132import org.apache.hadoop.ipc.RemoteException;
133import org.apache.hadoop.security.UserGroupInformation;
134import org.apache.hadoop.util.ReflectionUtils;
135import org.apache.hadoop.util.Tool;
136import org.apache.hadoop.util.ToolRunner;
137import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
138import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
139import org.apache.yetus.audience.InterfaceAudience;
140import org.apache.yetus.audience.InterfaceStability;
141import org.apache.zookeeper.KeeperException;
142import org.slf4j.Logger;
143import org.slf4j.LoggerFactory;
144
145import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
146import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
147import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
148
149import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
150import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
151
152/**
153 * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
154 * table integrity problems in a corrupted HBase. This tool was written for hbase-1.x. It does not
155 * work with hbase-2.x; it can read state but is not allowed to change state; i.e. effect 'repair'.
156 * Even though it can 'read' state, given how so much has changed in how hbase1 and hbase2 operate,
157 * it will often misread. See hbck2 (HBASE-19121) for a hbck tool for hbase2. This class is
158 * deprecated.
159 *
160 * <p>
161 * Region consistency checks verify that hbase:meta, region deployment on region
162 * servers and the state of data in HDFS (.regioninfo files) all are in
163 * accordance.
164 * <p>
165 * Table integrity checks verify that all possible row keys resolve to exactly
166 * one region of a table.  This means there are no individual degenerate
167 * or backwards regions; no holes between regions; and that there are no
168 * overlapping regions.
169 * <p>
170 * The general repair strategy works in two phases:
171 * <ol>
172 * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
173 * <li> Repair Region Consistency with hbase:meta and assignments
174 * </ol>
175 * <p>
176 * For table integrity repairs, the tables' region directories are scanned
177 * for .regioninfo files.  Each table's integrity is then verified.  If there
178 * are any orphan regions (regions with no .regioninfo files) or holes, new
179 * regions are fabricated.  Backwards regions are sidelined as well as empty
180 * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
181 * a new region is created and all data is merged into the new region.
182 * <p>
183 * Table integrity repairs deal solely with HDFS and could potentially be done
184 * offline -- the hbase region servers or master do not need to be running.
185 * This phase can eventually be used to completely reconstruct the hbase:meta table in
186 * an offline fashion.
187 * <p>
188 * Region consistency requires three conditions -- 1) valid .regioninfo file
189 * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
190 * and 3) a region is deployed only at the regionserver that was assigned to
191 * with proper state in the master.
192 * <p>
193 * Region consistency repairs require hbase to be online so that hbck can
194 * contact the HBase master and region servers.  The hbck#connect() method must
195 * first be called successfully.  Much of the region consistency information
196 * is transient and less risky to repair.
197 * <p>
198 * If hbck is run from the command line, there are a handful of arguments that
199 * can be used to limit the kinds of repairs hbck will do.  See the code in
200 * {@link #printUsageAndExit()} for more details.
201 * @deprecated For removal in hbase-4.0.0. Use HBCK2 instead.
202 */
203@Deprecated
204@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
205@InterfaceStability.Evolving
206public class HBaseFsck extends Configured implements Closeable {
207  public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
208  public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
209  private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
210  private static boolean rsSupportsOffline = true;
211  private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
212  private static final int DEFAULT_MAX_MERGE = 5;
213
214  /**
215   * Here is where hbase-1.x used to default the lock for hbck1.
216   * It puts in place a lock when it goes to write/make changes.
217   */
218  @VisibleForTesting
219  public static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
220  private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5;
221  private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds
222  private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds
223  // We have to set the timeout value > HdfsConstants.LEASE_SOFTLIMIT_PERIOD.
224  // In HADOOP-2.6 and later, the Namenode proxy now created with custom RetryPolicy for
225  // AlreadyBeingCreatedException which is implies timeout on this operations up to
226  // HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds).
227  private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds
228  private static final int DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS = 5;
229  private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds
230  private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds
231
232  /**********************
233   * Internal resources
234   **********************/
235  private static final Logger LOG = LoggerFactory.getLogger(HBaseFsck.class.getName());
236  private ClusterMetrics status;
237  private ClusterConnection connection;
238  private Admin admin;
239  private Table meta;
240  // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
241  protected ExecutorService executor;
242  private long startMillis = EnvironmentEdgeManager.currentTime();
243  private HFileCorruptionChecker hfcc;
244  private int retcode = 0;
245  private Path HBCK_LOCK_PATH;
246  private FSDataOutputStream hbckOutFd;
247  // This lock is to prevent cleanup of balancer resources twice between
248  // ShutdownHook and the main code. We cleanup only if the connect() is
249  // successful
250  private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
251
252  // Unsupported options in HBase 2.0+
253  private static final Set<String> unsupportedOptionsInV2 = Sets.newHashSet("-fix",
254      "-fixAssignments", "-fixMeta", "-fixHdfsHoles", "-fixHdfsOrphans", "-fixTableOrphans",
255      "-fixHdfsOverlaps", "-sidelineBigOverlaps", "-fixSplitParents", "-removeParents",
256      "-fixEmptyMetaCells", "-repair", "-repairHoles", "-maxOverlapsToSideline", "-maxMerge");
257
258  /***********
259   * Options
260   ***********/
261  private static boolean details = false; // do we display the full report
262  private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
263  private static boolean forceExclusive = false; // only this hbck can modify HBase
264  private boolean fixAssignments = false; // fix assignment errors?
265  private boolean fixMeta = false; // fix meta errors?
266  private boolean checkHdfs = true; // load and check fs consistency?
267  private boolean fixHdfsHoles = false; // fix fs holes?
268  private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
269  private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
270  private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
271  private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
272  private boolean fixSplitParents = false; // fix lingering split parents
273  private boolean removeParents = false; // remove split parents
274  private boolean fixReferenceFiles = false; // fix lingering reference store file
275  private boolean fixHFileLinks = false; // fix lingering HFileLinks
276  private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
277  private boolean fixReplication = false; // fix undeleted replication queues for removed peer
278  private boolean cleanReplicationBarrier = false; // clean replication barriers of a table
279  private boolean fixAny = false; // Set to true if any of the fix is required.
280
281  // limit checking/fixes to listed tables, if empty attempt to check/fix all
282  // hbase:meta are always checked
283  private Set<TableName> tablesIncluded = new HashSet<>();
284  private TableName cleanReplicationBarrierTable;
285  private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
286  // maximum number of overlapping regions to sideline
287  private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE;
288  private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
289  private Path sidelineDir = null;
290
291  private boolean rerun = false; // if we tried to fix something, rerun hbck
292  private static boolean summary = false; // if we want to print less output
293  private boolean checkMetaOnly = false;
294  private boolean checkRegionBoundaries = false;
295  private boolean ignorePreCheckPermission = false; // if pre-check permission
296
297  /*********
298   * State
299   *********/
300  final private HbckErrorReporter errors;
301  int fixes = 0;
302
303  /**
304   * This map contains the state of all hbck items.  It maps from encoded region
305   * name to HbckRegionInfo structure.  The information contained in HbckRegionInfo is used
306   * to detect and correct consistency (hdfs/meta/deployment) problems.
307   */
308  private TreeMap<String, HbckRegionInfo> regionInfoMap = new TreeMap<>();
309  // Empty regioninfo qualifiers in hbase:meta
310  private Set<Result> emptyRegionInfoQualifiers = new HashSet<>();
311
312  /**
313   * This map from Tablename -> TableInfo contains the structures necessary to
314   * detect table consistency problems (holes, dupes, overlaps).  It is sorted
315   * to prevent dupes.
316   *
317   * If tablesIncluded is empty, this map contains all tables.
318   * Otherwise, it contains only meta tables and tables in tablesIncluded,
319   * unless checkMetaOnly is specified, in which case, it contains only
320   * the meta table
321   */
322  private SortedMap<TableName, HbckTableInfo> tablesInfo = new ConcurrentSkipListMap<>();
323
324  /**
325   * When initially looking at HDFS, we attempt to find any orphaned data.
326   */
327  private List<HbckRegionInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<>());
328
329  private Map<TableName, Set<String>> orphanTableDirs = new HashMap<>();
330  private Map<TableName, TableState> tableStates = new HashMap<>();
331  private final RetryCounterFactory lockFileRetryCounterFactory;
332  private final RetryCounterFactory createZNodeRetryCounterFactory;
333
334  private Map<TableName, Set<String>> skippedRegions = new HashMap<>();
335
336  private ZKWatcher zkw = null;
337  private String hbckEphemeralNodePath = null;
338  private boolean hbckZodeCreated = false;
339
340  /**
341   * Constructor
342   *
343   * @param conf Configuration object
344   * @throws MasterNotRunningException if the master is not running
345   * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
346   */
347  public HBaseFsck(Configuration conf) throws IOException, ClassNotFoundException {
348    this(conf, createThreadPool(conf));
349  }
350
351  private static ExecutorService createThreadPool(Configuration conf) {
352    int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
353    return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
354  }
355
356  /**
357   * Constructor
358   *
359   * @param conf
360   *          Configuration object
361   * @throws MasterNotRunningException
362   *           if the master is not running
363   * @throws ZooKeeperConnectionException
364   *           if unable to connect to ZooKeeper
365   */
366  public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
367      ZooKeeperConnectionException, IOException, ClassNotFoundException {
368    super(conf);
369    errors = getErrorReporter(getConf());
370    this.executor = exec;
371    lockFileRetryCounterFactory = createLockRetryCounterFactory(getConf());
372    createZNodeRetryCounterFactory = createZnodeRetryCounterFactory(getConf());
373    zkw = createZooKeeperWatcher();
374  }
375
376  /**
377   * @return A retry counter factory configured for retrying lock file creation.
378   */
379  public static RetryCounterFactory createLockRetryCounterFactory(Configuration conf) {
380    return new RetryCounterFactory(
381        conf.getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
382        conf.getInt("hbase.hbck.lockfile.attempt.sleep.interval",
383            DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),
384        conf.getInt("hbase.hbck.lockfile.attempt.maxsleeptime",
385            DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));
386  }
387
388  /**
389   * @return A retry counter factory configured for retrying znode creation.
390   */
391  private static RetryCounterFactory createZnodeRetryCounterFactory(Configuration conf) {
392    return new RetryCounterFactory(
393        conf.getInt("hbase.hbck.createznode.attempts", DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS),
394        conf.getInt("hbase.hbck.createznode.attempt.sleep.interval",
395            DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL),
396        conf.getInt("hbase.hbck.createznode.attempt.maxsleeptime",
397            DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME));
398  }
399
400  /**
401   * @return Return the tmp dir this tool writes too.
402   */
403  @VisibleForTesting
404  public static Path getTmpDir(Configuration conf) throws IOException {
405    return new Path(FSUtils.getRootDir(conf), HConstants.HBASE_TEMP_DIRECTORY);
406  }
407
408  private static class FileLockCallable implements Callable<FSDataOutputStream> {
409    RetryCounter retryCounter;
410    private final Configuration conf;
411    private Path hbckLockPath = null;
412
413    public FileLockCallable(Configuration conf, RetryCounter retryCounter) {
414      this.retryCounter = retryCounter;
415      this.conf = conf;
416    }
417
418    /**
419     * @return Will be <code>null</code> unless you call {@link #call()}
420     */
421    Path getHbckLockPath() {
422      return this.hbckLockPath;
423    }
424
425    @Override
426    public FSDataOutputStream call() throws IOException {
427      try {
428        FileSystem fs = FSUtils.getCurrentFileSystem(this.conf);
429        FsPermission defaultPerms = FSUtils.getFilePermissions(fs, this.conf,
430            HConstants.DATA_FILE_UMASK_KEY);
431        Path tmpDir = getTmpDir(conf);
432        this.hbckLockPath = new Path(tmpDir, HBCK_LOCK_FILE);
433        fs.mkdirs(tmpDir);
434        final FSDataOutputStream out = createFileWithRetries(fs, this.hbckLockPath, defaultPerms);
435        out.writeBytes(InetAddress.getLocalHost().toString());
436        // Add a note into the file we write on why hbase2 is writing out an hbck1 lock file.
437        out.writeBytes(" Written by an hbase-2.x Master to block an " +
438            "attempt by an hbase-1.x HBCK tool making modification to state. " +
439            "See 'HBCK must match HBase server version' in the hbase refguide.");
440        out.flush();
441        return out;
442      } catch(RemoteException e) {
443        if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
444          return null;
445        } else {
446          throw e;
447        }
448      }
449    }
450
451    private FSDataOutputStream createFileWithRetries(final FileSystem fs,
452        final Path hbckLockFilePath, final FsPermission defaultPerms)
453        throws IOException {
454      IOException exception = null;
455      do {
456        try {
457          return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false);
458        } catch (IOException ioe) {
459          LOG.info("Failed to create lock file " + hbckLockFilePath.getName()
460              + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of "
461              + retryCounter.getMaxAttempts());
462          LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(),
463              ioe);
464          try {
465            exception = ioe;
466            retryCounter.sleepUntilNextRetry();
467          } catch (InterruptedException ie) {
468            throw (InterruptedIOException) new InterruptedIOException(
469                "Can't create lock file " + hbckLockFilePath.getName())
470            .initCause(ie);
471          }
472        }
473      } while (retryCounter.shouldRetry());
474
475      throw exception;
476    }
477  }
478
479  /**
480   * This method maintains a lock using a file. If the creation fails we return null
481   *
482   * @return FSDataOutputStream object corresponding to the newly opened lock file
483   * @throws IOException if IO failure occurs
484   */
485  public static Pair<Path, FSDataOutputStream> checkAndMarkRunningHbck(Configuration conf,
486      RetryCounter retryCounter) throws IOException {
487    FileLockCallable callable = new FileLockCallable(conf, retryCounter);
488    ExecutorService executor = Executors.newFixedThreadPool(1);
489    FutureTask<FSDataOutputStream> futureTask = new FutureTask<>(callable);
490    executor.execute(futureTask);
491    final int timeoutInSeconds = conf.getInt(
492      "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT);
493    FSDataOutputStream stream = null;
494    try {
495      stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);
496    } catch (ExecutionException ee) {
497      LOG.warn("Encountered exception when opening lock file", ee);
498    } catch (InterruptedException ie) {
499      LOG.warn("Interrupted when opening lock file", ie);
500      Thread.currentThread().interrupt();
501    } catch (TimeoutException exception) {
502      // took too long to obtain lock
503      LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");
504      futureTask.cancel(true);
505    } finally {
506      executor.shutdownNow();
507    }
508    return new Pair<Path, FSDataOutputStream>(callable.getHbckLockPath(), stream);
509  }
510
511  private void unlockHbck() {
512    if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) {
513      RetryCounter retryCounter = lockFileRetryCounterFactory.create();
514      do {
515        try {
516          IOUtils.closeQuietly(hbckOutFd);
517          FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()), HBCK_LOCK_PATH, true);
518          LOG.info("Finishing hbck");
519          return;
520        } catch (IOException ioe) {
521          LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try="
522              + (retryCounter.getAttemptTimes() + 1) + " of "
523              + retryCounter.getMaxAttempts());
524          LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe);
525          try {
526            retryCounter.sleepUntilNextRetry();
527          } catch (InterruptedException ie) {
528            Thread.currentThread().interrupt();
529            LOG.warn("Interrupted while deleting lock file" +
530                HBCK_LOCK_PATH);
531            return;
532          }
533        }
534      } while (retryCounter.shouldRetry());
535    }
536  }
537
538  /**
539   * To repair region consistency, one must call connect() in order to repair
540   * online state.
541   */
542  public void connect() throws IOException {
543
544    if (isExclusive()) {
545      // Grab the lock
546      Pair<Path, FSDataOutputStream> pair =
547          checkAndMarkRunningHbck(getConf(), this.lockFileRetryCounterFactory.create());
548      HBCK_LOCK_PATH = pair.getFirst();
549      this.hbckOutFd = pair.getSecond();
550      if (hbckOutFd == null) {
551        setRetCode(-1);
552        LOG.error("Another instance of hbck is fixing HBase, exiting this instance. " +
553            "[If you are sure no other instance is running, delete the lock file " +
554            HBCK_LOCK_PATH + " and rerun the tool]");
555        throw new IOException("Duplicate hbck - Abort");
556      }
557
558      // Make sure to cleanup the lock
559      hbckLockCleanup.set(true);
560    }
561
562
563    // Add a shutdown hook to this thread, in case user tries to
564    // kill the hbck with a ctrl-c, we want to cleanup the lock so that
565    // it is available for further calls
566    Runtime.getRuntime().addShutdownHook(new Thread() {
567      @Override
568      public void run() {
569        IOUtils.closeQuietly(HBaseFsck.this);
570        cleanupHbckZnode();
571        unlockHbck();
572      }
573    });
574
575    LOG.info("Launching hbck");
576
577    connection = (ClusterConnection)ConnectionFactory.createConnection(getConf());
578    admin = connection.getAdmin();
579    meta = connection.getTable(TableName.META_TABLE_NAME);
580    status = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS,
581      Option.DEAD_SERVERS, Option.MASTER, Option.BACKUP_MASTERS,
582      Option.REGIONS_IN_TRANSITION, Option.HBASE_VERSION));
583  }
584
585  /**
586   * Get deployed regions according to the region servers.
587   */
588  private void loadDeployedRegions() throws IOException, InterruptedException {
589    // From the master, get a list of all known live region servers
590    Collection<ServerName> regionServers = status.getLiveServerMetrics().keySet();
591    errors.print("Number of live region servers: " + regionServers.size());
592    if (details) {
593      for (ServerName rsinfo: regionServers) {
594        errors.print("  " + rsinfo.getServerName());
595      }
596    }
597
598    // From the master, get a list of all dead region servers
599    Collection<ServerName> deadRegionServers = status.getDeadServerNames();
600    errors.print("Number of dead region servers: " + deadRegionServers.size());
601    if (details) {
602      for (ServerName name: deadRegionServers) {
603        errors.print("  " + name);
604      }
605    }
606
607    // Print the current master name and state
608    errors.print("Master: " + status.getMasterName());
609
610    // Print the list of all backup masters
611    Collection<ServerName> backupMasters = status.getBackupMasterNames();
612    errors.print("Number of backup masters: " + backupMasters.size());
613    if (details) {
614      for (ServerName name: backupMasters) {
615        errors.print("  " + name);
616      }
617    }
618
619    errors.print("Average load: " + status.getAverageLoad());
620    errors.print("Number of requests: " + status.getRequestCount());
621    errors.print("Number of regions: " + status.getRegionCount());
622
623    List<RegionState> rits = status.getRegionStatesInTransition();
624    errors.print("Number of regions in transition: " + rits.size());
625    if (details) {
626      for (RegionState state: rits) {
627        errors.print("  " + state.toDescriptiveString());
628      }
629    }
630
631    // Determine what's deployed
632    processRegionServers(regionServers);
633  }
634
635  /**
636   * Clear the current state of hbck.
637   */
638  private void clearState() {
639    // Make sure regionInfo is empty before starting
640    fixes = 0;
641    regionInfoMap.clear();
642    emptyRegionInfoQualifiers.clear();
643    tableStates.clear();
644    errors.clear();
645    tablesInfo.clear();
646    orphanHdfsDirs.clear();
647    skippedRegions.clear();
648  }
649
650  /**
651   * This repair method analyzes hbase data in hdfs and repairs it to satisfy
652   * the table integrity rules.  HBase doesn't need to be online for this
653   * operation to work.
654   */
655  public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
656    // Initial pass to fix orphans.
657    if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
658        || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
659      LOG.info("Loading regioninfos HDFS");
660      // if nothing is happening this should always complete in two iterations.
661      int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
662      int curIter = 0;
663      do {
664        clearState(); // clears hbck state and reset fixes to 0 and.
665        // repair what's on HDFS
666        restoreHdfsIntegrity();
667        curIter++;// limit the number of iterations.
668      } while (fixes > 0 && curIter <= maxIterations);
669
670      // Repairs should be done in the first iteration and verification in the second.
671      // If there are more than 2 passes, something funny has happened.
672      if (curIter > 2) {
673        if (curIter == maxIterations) {
674          LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
675              + "Tables integrity may not be fully repaired!");
676        } else {
677          LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
678        }
679      }
680    }
681  }
682
683  /**
684   * This repair method requires the cluster to be online since it contacts
685   * region servers and the masters.  It makes each region's state in HDFS, in
686   * hbase:meta, and deployments consistent.
687   *
688   * @return If &gt; 0 , number of errors detected, if &lt; 0 there was an unrecoverable
689   *     error.  If 0, we have a clean hbase.
690   */
691  public int onlineConsistencyRepair() throws IOException, KeeperException,
692    InterruptedException {
693
694    // get regions according to what is online on each RegionServer
695    loadDeployedRegions();
696    // check whether hbase:meta is deployed and online
697    recordMetaRegion();
698    // Check if hbase:meta is found only once and in the right place
699    if (!checkMetaRegion()) {
700      String errorMsg = "hbase:meta table is not consistent. ";
701      if (shouldFixAssignments()) {
702        errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
703      } else {
704        errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
705      }
706      errors.reportError(errorMsg + " Exiting...");
707      return -2;
708    }
709    // Not going with further consistency check for tables when hbase:meta itself is not consistent.
710    LOG.info("Loading regionsinfo from the hbase:meta table");
711    boolean success = loadMetaEntries();
712    if (!success) return -1;
713
714    // Empty cells in hbase:meta?
715    reportEmptyMetaCells();
716
717    // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
718    if (shouldFixEmptyMetaCells()) {
719      fixEmptyMetaCells();
720    }
721
722    // get a list of all tables that have not changed recently.
723    if (!checkMetaOnly) {
724      reportTablesInFlux();
725    }
726
727    // Get disabled tables states
728    loadTableStates();
729
730    // load regiondirs and regioninfos from HDFS
731    if (shouldCheckHdfs()) {
732      LOG.info("Loading region directories from HDFS");
733      loadHdfsRegionDirs();
734      LOG.info("Loading region information from HDFS");
735      loadHdfsRegionInfos();
736    }
737
738    // fix the orphan tables
739    fixOrphanTables();
740
741    LOG.info("Checking and fixing region consistency");
742    // Check and fix consistency
743    checkAndFixConsistency();
744
745    // Check integrity (does not fix)
746    checkIntegrity();
747    return errors.getErrorList().size();
748  }
749
750  /**
751   * This method maintains an ephemeral znode. If the creation fails we return false or throw
752   * exception
753   *
754   * @return true if creating znode succeeds; false otherwise
755   * @throws IOException if IO failure occurs
756   */
757  private boolean setMasterInMaintenanceMode() throws IOException {
758    RetryCounter retryCounter = createZNodeRetryCounterFactory.create();
759    hbckEphemeralNodePath = ZNodePaths.joinZNode(
760      zkw.getZNodePaths().masterMaintZNode,
761      "hbck-" + Long.toString(EnvironmentEdgeManager.currentTime()));
762    do {
763      try {
764        hbckZodeCreated = ZKUtil.createEphemeralNodeAndWatch(zkw, hbckEphemeralNodePath, null);
765        if (hbckZodeCreated) {
766          break;
767        }
768      } catch (KeeperException e) {
769        if (retryCounter.getAttemptTimes() >= retryCounter.getMaxAttempts()) {
770           throw new IOException("Can't create znode " + hbckEphemeralNodePath, e);
771        }
772        // fall through and retry
773      }
774
775      LOG.warn("Fail to create znode " + hbckEphemeralNodePath + ", try=" +
776          (retryCounter.getAttemptTimes() + 1) + " of " + retryCounter.getMaxAttempts());
777
778      try {
779        retryCounter.sleepUntilNextRetry();
780      } catch (InterruptedException ie) {
781        throw (InterruptedIOException) new InterruptedIOException(
782              "Can't create znode " + hbckEphemeralNodePath).initCause(ie);
783      }
784    } while (retryCounter.shouldRetry());
785    return hbckZodeCreated;
786  }
787
788  private void cleanupHbckZnode() {
789    try {
790      if (zkw != null && hbckZodeCreated) {
791        ZKUtil.deleteNode(zkw, hbckEphemeralNodePath);
792        hbckZodeCreated = false;
793      }
794    } catch (KeeperException e) {
795      // Ignore
796      if (!e.code().equals(KeeperException.Code.NONODE)) {
797        LOG.warn("Delete HBCK znode " + hbckEphemeralNodePath + " failed ", e);
798      }
799    }
800  }
801
802  /**
803   * Contacts the master and prints out cluster-wide information
804   * @return 0 on success, non-zero on failure
805   */
806  public int onlineHbck()
807      throws IOException, KeeperException, InterruptedException, ReplicationException {
808    // print hbase server version
809    errors.print("Version: " + status.getHBaseVersion());
810
811    // Clean start
812    clearState();
813    // Do offline check and repair first
814    offlineHdfsIntegrityRepair();
815    offlineReferenceFileRepair();
816    offlineHLinkFileRepair();
817    // If Master runs maintenance tasks (such as balancer, catalog janitor, etc) during online
818    // hbck, it is likely that hbck would be misled and report transient errors.  Therefore, it
819    // is better to set Master into maintenance mode during online hbck.
820    //
821    if (!setMasterInMaintenanceMode()) {
822      LOG.warn("HBCK is running while master is not in maintenance mode, you might see transient "
823        + "error.  Please run HBCK multiple times to reduce the chance of transient error.");
824    }
825
826    onlineConsistencyRepair();
827
828    if (checkRegionBoundaries) {
829      checkRegionBoundaries();
830    }
831
832    checkAndFixReplication();
833
834    cleanReplicationBarrier();
835
836    // Remove the hbck znode
837    cleanupHbckZnode();
838
839    // Remove the hbck lock
840    unlockHbck();
841
842    // Print table summary
843    printTableSummary(tablesInfo);
844    return errors.summarize();
845  }
846
847  public static byte[] keyOnly(byte[] b) {
848    if (b == null)
849      return b;
850    int rowlength = Bytes.toShort(b, 0);
851    byte[] result = new byte[rowlength];
852    System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
853    return result;
854  }
855
856  @Override
857  public void close() throws IOException {
858    try {
859      cleanupHbckZnode();
860      unlockHbck();
861    } catch (Exception io) {
862      LOG.warn(io.toString(), io);
863    } finally {
864      if (zkw != null) {
865        zkw.close();
866        zkw = null;
867      }
868      IOUtils.closeQuietly(admin);
869      IOUtils.closeQuietly(meta);
870      IOUtils.closeQuietly(connection);
871    }
872  }
873
874  private static class RegionBoundariesInformation {
875    public byte [] regionName;
876    public byte [] metaFirstKey;
877    public byte [] metaLastKey;
878    public byte [] storesFirstKey;
879    public byte [] storesLastKey;
880    @Override
881    public String toString () {
882      return "regionName=" + Bytes.toStringBinary(regionName) +
883             "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
884             "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
885             "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
886             "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
887    }
888  }
889
890  public void checkRegionBoundaries() {
891    try {
892      ByteArrayComparator comparator = new ByteArrayComparator();
893      List<RegionInfo> regions = MetaTableAccessor.getAllRegions(connection, true);
894      final RegionBoundariesInformation currentRegionBoundariesInformation =
895          new RegionBoundariesInformation();
896      Path hbaseRoot = FSUtils.getRootDir(getConf());
897      for (RegionInfo regionInfo : regions) {
898        Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
899        currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
900        // For each region, get the start and stop key from the META and compare them to the
901        // same information from the Stores.
902        Path path = new Path(tableDir, regionInfo.getEncodedName());
903        FileSystem fs = path.getFileSystem(getConf());
904        FileStatus[] files = fs.listStatus(path);
905        // For all the column families in this region...
906        byte[] storeFirstKey = null;
907        byte[] storeLastKey = null;
908        for (FileStatus file : files) {
909          String fileName = file.getPath().toString();
910          fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
911          if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
912            FileStatus[] storeFiles = fs.listStatus(file.getPath());
913            // For all the stores in this column family.
914            for (FileStatus storeFile : storeFiles) {
915              HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(),
916                CacheConfig.DISABLED, true, getConf());
917              if ((reader.getFirstKey() != null)
918                  && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
919                      ((KeyValue.KeyOnlyKeyValue) reader.getFirstKey().get()).getKey()) > 0))) {
920                storeFirstKey = ((KeyValue.KeyOnlyKeyValue)reader.getFirstKey().get()).getKey();
921              }
922              if ((reader.getLastKey() != null)
923                  && ((storeLastKey == null) || (comparator.compare(storeLastKey,
924                      ((KeyValue.KeyOnlyKeyValue)reader.getLastKey().get()).getKey())) < 0)) {
925                storeLastKey = ((KeyValue.KeyOnlyKeyValue)reader.getLastKey().get()).getKey();
926              }
927              reader.close();
928            }
929          }
930        }
931        currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
932        currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
933        currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
934        currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
935        if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
936          currentRegionBoundariesInformation.metaFirstKey = null;
937        if (currentRegionBoundariesInformation.metaLastKey.length == 0)
938          currentRegionBoundariesInformation.metaLastKey = null;
939
940        // For a region to be correct, we need the META start key to be smaller or equal to the
941        // smallest start key from all the stores, and the start key from the next META entry to
942        // be bigger than the last key from all the current stores. First region start key is null;
943        // Last region end key is null; some regions can be empty and not have any store.
944
945        boolean valid = true;
946        // Checking start key.
947        if ((currentRegionBoundariesInformation.storesFirstKey != null)
948            && (currentRegionBoundariesInformation.metaFirstKey != null)) {
949          valid = valid
950              && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
951                currentRegionBoundariesInformation.metaFirstKey) >= 0;
952        }
953        // Checking stop key.
954        if ((currentRegionBoundariesInformation.storesLastKey != null)
955            && (currentRegionBoundariesInformation.metaLastKey != null)) {
956          valid = valid
957              && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
958                currentRegionBoundariesInformation.metaLastKey) < 0;
959        }
960        if (!valid) {
961          errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
962            tablesInfo.get(regionInfo.getTable()));
963          LOG.warn("Region's boundaries not aligned between stores and META for:");
964          LOG.warn(Objects.toString(currentRegionBoundariesInformation));
965        }
966      }
967    } catch (IOException e) {
968      LOG.error(e.toString(), e);
969    }
970  }
971
972  /**
973   * Iterates through the list of all orphan/invalid regiondirs.
974   */
975  private void adoptHdfsOrphans(Collection<HbckRegionInfo> orphanHdfsDirs) throws IOException {
976    for (HbckRegionInfo hi : orphanHdfsDirs) {
977      LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
978      adoptHdfsOrphan(hi);
979    }
980  }
981
982  /**
983   * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
984   * these orphans by creating a new region, and moving the column families,
985   * recovered edits, WALs, into the new region dir.  We determine the region
986   * startkey and endkeys by looking at all of the hfiles inside the column
987   * families to identify the min and max keys. The resulting region will
988   * likely violate table integrity but will be dealt with by merging
989   * overlapping regions.
990   */
991  @SuppressWarnings("deprecation")
992  private void adoptHdfsOrphan(HbckRegionInfo hi) throws IOException {
993    Path p = hi.getHdfsRegionDir();
994    FileSystem fs = p.getFileSystem(getConf());
995    FileStatus[] dirs = fs.listStatus(p);
996    if (dirs == null) {
997      LOG.warn("Attempt to adopt orphan hdfs region skipped because no files present in " +
998          p + ". This dir could probably be deleted.");
999      return ;
1000    }
1001
1002    TableName tableName = hi.getTableName();
1003    HbckTableInfo tableInfo = tablesInfo.get(tableName);
1004    Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
1005    TableDescriptor template = tableInfo.getTableDescriptor();
1006
1007    // find min and max key values
1008    Pair<byte[],byte[]> orphanRegionRange = null;
1009    for (FileStatus cf : dirs) {
1010      String cfName= cf.getPath().getName();
1011      // TODO Figure out what the special dirs are
1012      if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
1013
1014      FileStatus[] hfiles = fs.listStatus(cf.getPath());
1015      for (FileStatus hfile : hfiles) {
1016        byte[] start, end;
1017        HFile.Reader hf = null;
1018        try {
1019          hf = HFile.createReader(fs, hfile.getPath(), CacheConfig.DISABLED, true, getConf());
1020          hf.loadFileInfo();
1021          Optional<Cell> startKv = hf.getFirstKey();
1022          start = CellUtil.cloneRow(startKv.get());
1023          Optional<Cell> endKv = hf.getLastKey();
1024          end = CellUtil.cloneRow(endKv.get());
1025        } catch (IOException ioe) {
1026          LOG.warn("Problem reading orphan file " + hfile + ", skipping");
1027          continue;
1028        } catch (NullPointerException ioe) {
1029          LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
1030          continue;
1031        } finally {
1032          if (hf != null) {
1033            hf.close();
1034          }
1035        }
1036
1037        // expand the range to include the range of all hfiles
1038        if (orphanRegionRange == null) {
1039          // first range
1040          orphanRegionRange = new Pair<>(start, end);
1041        } else {
1042          // TODO add test
1043
1044          // expand range only if the hfile is wider.
1045          if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
1046            orphanRegionRange.setFirst(start);
1047          }
1048          if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
1049            orphanRegionRange.setSecond(end);
1050          }
1051        }
1052      }
1053    }
1054    if (orphanRegionRange == null) {
1055      LOG.warn("No data in dir " + p + ", sidelining data");
1056      fixes++;
1057      sidelineRegionDir(fs, hi);
1058      return;
1059    }
1060    LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
1061        Bytes.toString(orphanRegionRange.getSecond()) + ")");
1062
1063    // create new region on hdfs. move data into place.
1064    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(template.getTableName())
1065        .setStartKey(orphanRegionRange.getFirst())
1066        .setEndKey(Bytes.add(orphanRegionRange.getSecond(), new byte[1]))
1067        .build();
1068    LOG.info("Creating new region : " + regionInfo);
1069    HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), regionInfo, template);
1070    Path target = region.getRegionFileSystem().getRegionDir();
1071
1072    // rename all the data to new region
1073    mergeRegionDirs(target, hi);
1074    fixes++;
1075  }
1076
1077  /**
1078   * This method determines if there are table integrity errors in HDFS.  If
1079   * there are errors and the appropriate "fix" options are enabled, the method
1080   * will first correct orphan regions making them into legit regiondirs, and
1081   * then reload to merge potentially overlapping regions.
1082   *
1083   * @return number of table integrity errors found
1084   */
1085  private int restoreHdfsIntegrity() throws IOException, InterruptedException {
1086    // Determine what's on HDFS
1087    LOG.info("Loading HBase regioninfo from HDFS...");
1088    loadHdfsRegionDirs(); // populating regioninfo table.
1089
1090    int errs = errors.getErrorList().size();
1091    // First time just get suggestions.
1092    tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1093    checkHdfsIntegrity(false, false);
1094
1095    if (errors.getErrorList().size() == errs) {
1096      LOG.info("No integrity errors.  We are done with this phase. Glorious.");
1097      return 0;
1098    }
1099
1100    if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
1101      adoptHdfsOrphans(orphanHdfsDirs);
1102      // TODO optimize by incrementally adding instead of reloading.
1103    }
1104
1105    // Make sure there are no holes now.
1106    if (shouldFixHdfsHoles()) {
1107      clearState(); // this also resets # fixes.
1108      loadHdfsRegionDirs();
1109      tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1110      tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
1111    }
1112
1113    // Now we fix overlaps
1114    if (shouldFixHdfsOverlaps()) {
1115      // second pass we fix overlaps.
1116      clearState(); // this also resets # fixes.
1117      loadHdfsRegionDirs();
1118      tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1119      tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
1120    }
1121
1122    return errors.getErrorList().size();
1123  }
1124
1125  /**
1126   * Scan all the store file names to find any lingering reference files,
1127   * which refer to some none-exiting files. If "fix" option is enabled,
1128   * any lingering reference file will be sidelined if found.
1129   * <p>
1130   * Lingering reference file prevents a region from opening. It has to
1131   * be fixed before a cluster can start properly.
1132   */
1133  private void offlineReferenceFileRepair() throws IOException, InterruptedException {
1134    clearState();
1135    Configuration conf = getConf();
1136    Path hbaseRoot = FSUtils.getRootDir(conf);
1137    FileSystem fs = hbaseRoot.getFileSystem(conf);
1138    LOG.info("Computing mapping of all store files");
1139    Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot,
1140      new FSUtils.ReferenceFileFilter(fs), executor, errors);
1141    errors.print("");
1142    LOG.info("Validating mapping using HDFS state");
1143    for (Path path: allFiles.values()) {
1144      Path referredToFile = StoreFileInfo.getReferredToFile(path);
1145      if (fs.exists(referredToFile)) continue;  // good, expected
1146
1147      // Found a lingering reference file
1148      errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
1149        "Found lingering reference file " + path);
1150      if (!shouldFixReferenceFiles()) continue;
1151
1152      // Now, trying to fix it since requested
1153      boolean success = false;
1154      String pathStr = path.toString();
1155
1156      // A reference file path should be like
1157      // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
1158      // Up 5 directories to get the root folder.
1159      // So the file will be sidelined to a similar folder structure.
1160      int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
1161      for (int i = 0; index > 0 && i < 5; i++) {
1162        index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
1163      }
1164      if (index > 0) {
1165        Path rootDir = getSidelineDir();
1166        Path dst = new Path(rootDir, pathStr.substring(index + 1));
1167        fs.mkdirs(dst.getParent());
1168        LOG.info("Trying to sideline reference file "
1169          + path + " to " + dst);
1170        setShouldRerun();
1171
1172        success = fs.rename(path, dst);
1173        debugLsr(dst);
1174
1175      }
1176      if (!success) {
1177        LOG.error("Failed to sideline reference file " + path);
1178      }
1179    }
1180  }
1181
1182  /**
1183   * Scan all the store file names to find any lingering HFileLink files,
1184   * which refer to some none-exiting files. If "fix" option is enabled,
1185   * any lingering HFileLink file will be sidelined if found.
1186   */
1187  private void offlineHLinkFileRepair() throws IOException, InterruptedException {
1188    Configuration conf = getConf();
1189    Path hbaseRoot = FSUtils.getRootDir(conf);
1190    FileSystem fs = hbaseRoot.getFileSystem(conf);
1191    LOG.info("Computing mapping of all link files");
1192    Map<String, Path> allFiles = FSUtils
1193        .getTableStoreFilePathMap(fs, hbaseRoot, new FSUtils.HFileLinkFilter(), executor, errors);
1194    errors.print("");
1195
1196    LOG.info("Validating mapping using HDFS state");
1197    for (Path path : allFiles.values()) {
1198      // building HFileLink object to gather locations
1199      HFileLink actualLink = HFileLink.buildFromHFileLinkPattern(conf, path);
1200      if (actualLink.exists(fs)) continue; // good, expected
1201
1202      // Found a lingering HFileLink
1203      errors.reportError(ERROR_CODE.LINGERING_HFILELINK, "Found lingering HFileLink " + path);
1204      if (!shouldFixHFileLinks()) continue;
1205
1206      // Now, trying to fix it since requested
1207      setShouldRerun();
1208
1209      // An HFileLink path should be like
1210      // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/linkedtable=linkedregionname-linkedhfilename
1211      // sidelineing will happen in the ${hbase.rootdir}/${sidelinedir} directory with the same folder structure.
1212      boolean success = sidelineFile(fs, hbaseRoot, path);
1213
1214      if (!success) {
1215        LOG.error("Failed to sideline HFileLink file " + path);
1216      }
1217
1218      // An HFileLink backreference path should be like
1219      // ${hbase.rootdir}/archive/data/namespace/table_name/region_id/family_name/.links-linkedhfilename
1220      // sidelineing will happen in the ${hbase.rootdir}/${sidelinedir} directory with the same folder structure.
1221      Path backRefPath = FileLink.getBackReferencesDir(HFileArchiveUtil
1222              .getStoreArchivePath(conf, HFileLink.getReferencedTableName(path.getName().toString()),
1223                  HFileLink.getReferencedRegionName(path.getName().toString()),
1224                  path.getParent().getName()),
1225          HFileLink.getReferencedHFileName(path.getName().toString()));
1226      success = sidelineFile(fs, hbaseRoot, backRefPath);
1227
1228      if (!success) {
1229        LOG.error("Failed to sideline HFileLink backreference file " + path);
1230      }
1231    }
1232  }
1233
1234  private boolean sidelineFile(FileSystem fs, Path hbaseRoot, Path path) throws IOException {
1235    URI uri = hbaseRoot.toUri().relativize(path.toUri());
1236    if (uri.isAbsolute()) return false;
1237    String relativePath = uri.getPath();
1238    Path rootDir = getSidelineDir();
1239    Path dst = new Path(rootDir, relativePath);
1240    boolean pathCreated = fs.mkdirs(dst.getParent());
1241    if (!pathCreated) {
1242      LOG.error("Failed to create path: " + dst.getParent());
1243      return false;
1244    }
1245    LOG.info("Trying to sideline file " + path + " to " + dst);
1246    return fs.rename(path, dst);
1247  }
1248
1249  /**
1250   * TODO -- need to add tests for this.
1251   */
1252  private void reportEmptyMetaCells() {
1253    errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
1254      emptyRegionInfoQualifiers.size());
1255    if (details) {
1256      for (Result r: emptyRegionInfoQualifiers) {
1257        errors.print("  " + r);
1258      }
1259    }
1260  }
1261
1262  /**
1263   * TODO -- need to add tests for this.
1264   */
1265  private void reportTablesInFlux() {
1266    AtomicInteger numSkipped = new AtomicInteger(0);
1267    TableDescriptor[] allTables = getTables(numSkipped);
1268    errors.print("Number of Tables: " + allTables.length);
1269    if (details) {
1270      if (numSkipped.get() > 0) {
1271        errors.detail("Number of Tables in flux: " + numSkipped.get());
1272      }
1273      for (TableDescriptor td : allTables) {
1274        errors.detail("  Table: " + td.getTableName() + "\t" +
1275                           (td.isReadOnly() ? "ro" : "rw") + "\t" +
1276                            (td.isMetaRegion() ? "META" : "    ") + "\t" +
1277                           " families: " + td.getColumnFamilyCount());
1278      }
1279    }
1280  }
1281
1282  public HbckErrorReporter getErrors() {
1283    return errors;
1284  }
1285
1286  /**
1287   * Populate hbi's from regionInfos loaded from file system.
1288   */
1289  private SortedMap<TableName, HbckTableInfo> loadHdfsRegionInfos()
1290      throws IOException, InterruptedException {
1291    tablesInfo.clear(); // regenerating the data
1292    // generate region split structure
1293    Collection<HbckRegionInfo> hbckRegionInfos = regionInfoMap.values();
1294
1295    // Parallelized read of .regioninfo files.
1296    List<WorkItemHdfsRegionInfo> hbis = new ArrayList<>(hbckRegionInfos.size());
1297    List<Future<Void>> hbiFutures;
1298
1299    for (HbckRegionInfo hbi : hbckRegionInfos) {
1300      WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1301      hbis.add(work);
1302    }
1303
1304    // Submit and wait for completion
1305    hbiFutures = executor.invokeAll(hbis);
1306
1307    for(int i=0; i<hbiFutures.size(); i++) {
1308      WorkItemHdfsRegionInfo work = hbis.get(i);
1309      Future<Void> f = hbiFutures.get(i);
1310      try {
1311        f.get();
1312      } catch(ExecutionException e) {
1313        LOG.warn("Failed to read .regioninfo file for region " +
1314              work.hbi.getRegionNameAsString(), e.getCause());
1315      }
1316    }
1317
1318    Path hbaseRoot = FSUtils.getRootDir(getConf());
1319    FileSystem fs = hbaseRoot.getFileSystem(getConf());
1320    // serialized table info gathering.
1321    for (HbckRegionInfo hbi: hbckRegionInfos) {
1322
1323      if (hbi.getHdfsHRI() == null) {
1324        // was an orphan
1325        continue;
1326      }
1327
1328
1329      // get table name from hdfs, populate various HBaseFsck tables.
1330      TableName tableName = hbi.getTableName();
1331      if (tableName == null) {
1332        // There was an entry in hbase:meta not in the HDFS?
1333        LOG.warn("tableName was null for: " + hbi);
1334        continue;
1335      }
1336
1337      HbckTableInfo modTInfo = tablesInfo.get(tableName);
1338      if (modTInfo == null) {
1339        // only executed once per table.
1340        modTInfo = new HbckTableInfo(tableName, this);
1341        tablesInfo.put(tableName, modTInfo);
1342        try {
1343          TableDescriptor htd =
1344              FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1345          modTInfo.htds.add(htd);
1346        } catch (IOException ioe) {
1347          if (!orphanTableDirs.containsKey(tableName)) {
1348            LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1349            //should only report once for each table
1350            errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1351                "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1352            Set<String> columns = new HashSet<>();
1353            orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1354          }
1355        }
1356      }
1357      if (!hbi.isSkipChecks()) {
1358        modTInfo.addRegionInfo(hbi);
1359      }
1360    }
1361
1362    loadTableInfosForTablesWithNoRegion();
1363    errors.print("");
1364
1365    return tablesInfo;
1366  }
1367
1368  /**
1369   * To get the column family list according to the column family dirs
1370   * @param columns
1371   * @param hbi
1372   * @return a set of column families
1373   * @throws IOException
1374   */
1375  private Set<String> getColumnFamilyList(Set<String> columns, HbckRegionInfo hbi)
1376      throws IOException {
1377    Path regionDir = hbi.getHdfsRegionDir();
1378    FileSystem fs = regionDir.getFileSystem(getConf());
1379    FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1380    for (FileStatus subdir : subDirs) {
1381      String columnfamily = subdir.getPath().getName();
1382      columns.add(columnfamily);
1383    }
1384    return columns;
1385  }
1386
1387  /**
1388   * To fabricate a .tableinfo file with following contents<br>
1389   * 1. the correct tablename <br>
1390   * 2. the correct colfamily list<br>
1391   * 3. the default properties for both {@link TableDescriptor} and {@link ColumnFamilyDescriptor}<br>
1392   * @throws IOException
1393   */
1394  private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1395      Set<String> columns) throws IOException {
1396    if (columns ==null || columns.isEmpty()) return false;
1397    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
1398    for (String columnfamimly : columns) {
1399      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(columnfamimly));
1400    }
1401    fstd.createTableDescriptor(builder.build(), true);
1402    return true;
1403  }
1404
1405  /**
1406   * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1407   * @throws IOException
1408   */
1409  public void fixEmptyMetaCells() throws IOException {
1410    if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1411      LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1412      for (Result region : emptyRegionInfoQualifiers) {
1413        deleteMetaRegion(region.getRow());
1414        errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1415      }
1416      emptyRegionInfoQualifiers.clear();
1417    }
1418  }
1419
1420  /**
1421   * To fix orphan table by creating a .tableinfo file under tableDir <br>
1422   * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1423   * 2. else create a default .tableinfo file with following items<br>
1424   * &nbsp;2.1 the correct tablename <br>
1425   * &nbsp;2.2 the correct colfamily list<br>
1426   * &nbsp;2.3 the default properties for both {@link TableDescriptor} and {@link ColumnFamilyDescriptor}<br>
1427   * @throws IOException
1428   */
1429  public void fixOrphanTables() throws IOException {
1430    if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1431
1432      List<TableName> tmpList = new ArrayList<>(orphanTableDirs.keySet().size());
1433      tmpList.addAll(orphanTableDirs.keySet());
1434      TableDescriptor[] htds = getTableDescriptors(tmpList);
1435      Iterator<Entry<TableName, Set<String>>> iter =
1436          orphanTableDirs.entrySet().iterator();
1437      int j = 0;
1438      int numFailedCase = 0;
1439      FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1440      while (iter.hasNext()) {
1441        Entry<TableName, Set<String>> entry =
1442            iter.next();
1443        TableName tableName = entry.getKey();
1444        LOG.info("Trying to fix orphan table error: " + tableName);
1445        if (j < htds.length) {
1446          if (tableName.equals(htds[j].getTableName())) {
1447            TableDescriptor htd = htds[j];
1448            LOG.info("fixing orphan table: " + tableName + " from cache");
1449            fstd.createTableDescriptor(htd, true);
1450            j++;
1451            iter.remove();
1452          }
1453        } else {
1454          if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1455            LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1456            LOG.warn("Strongly recommend to modify the TableDescriptor if necessary for: " + tableName);
1457            iter.remove();
1458          } else {
1459            LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1460            numFailedCase++;
1461          }
1462        }
1463        fixes++;
1464      }
1465
1466      if (orphanTableDirs.isEmpty()) {
1467        // all orphanTableDirs are luckily recovered
1468        // re-run doFsck after recovering the .tableinfo file
1469        setShouldRerun();
1470        LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1471      } else if (numFailedCase > 0) {
1472        LOG.error("Failed to fix " + numFailedCase
1473            + " OrphanTables with default .tableinfo files");
1474      }
1475
1476    }
1477    //cleanup the list
1478    orphanTableDirs.clear();
1479
1480  }
1481
1482  /**
1483   * Log an appropriate message about whether or not overlapping merges are computed in parallel.
1484   */
1485  private void logParallelMerge() {
1486    if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
1487      LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
1488          " false to run serially.");
1489    } else {
1490      LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
1491          " true to run in parallel.");
1492    }
1493  }
1494
1495  private SortedMap<TableName, HbckTableInfo> checkHdfsIntegrity(boolean fixHoles,
1496      boolean fixOverlaps) throws IOException {
1497    LOG.info("Checking HBase region split map from HDFS data...");
1498    logParallelMerge();
1499    for (HbckTableInfo tInfo : tablesInfo.values()) {
1500      TableIntegrityErrorHandler handler;
1501      if (fixHoles || fixOverlaps) {
1502        handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1503          fixHoles, fixOverlaps);
1504      } else {
1505        handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1506      }
1507      if (!tInfo.checkRegionChain(handler)) {
1508        // should dump info as well.
1509        errors.report("Found inconsistency in table " + tInfo.getName());
1510      }
1511    }
1512    return tablesInfo;
1513  }
1514
1515  Path getSidelineDir() throws IOException {
1516    if (sidelineDir == null) {
1517      Path hbaseDir = FSUtils.getRootDir(getConf());
1518      Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1519      sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1520          + startMillis);
1521    }
1522    return sidelineDir;
1523  }
1524
1525  /**
1526   * Sideline a region dir (instead of deleting it)
1527   */
1528  Path sidelineRegionDir(FileSystem fs, HbckRegionInfo hi) throws IOException {
1529    return sidelineRegionDir(fs, null, hi);
1530  }
1531
1532  /**
1533   * Sideline a region dir (instead of deleting it)
1534   *
1535   * @param parentDir if specified, the region will be sidelined to folder like
1536   *     {@literal .../parentDir/<table name>/<region name>}. The purpose is to group together
1537   *     similar regions sidelined, for example, those regions should be bulk loaded back later
1538   *     on. If NULL, it is ignored.
1539   */
1540  Path sidelineRegionDir(FileSystem fs,
1541      String parentDir, HbckRegionInfo hi) throws IOException {
1542    TableName tableName = hi.getTableName();
1543    Path regionDir = hi.getHdfsRegionDir();
1544
1545    if (!fs.exists(regionDir)) {
1546      LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1547      return null;
1548    }
1549
1550    Path rootDir = getSidelineDir();
1551    if (parentDir != null) {
1552      rootDir = new Path(rootDir, parentDir);
1553    }
1554    Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1555    Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1556    fs.mkdirs(sidelineRegionDir);
1557    boolean success = false;
1558    FileStatus[] cfs =  fs.listStatus(regionDir);
1559    if (cfs == null) {
1560      LOG.info("Region dir is empty: " + regionDir);
1561    } else {
1562      for (FileStatus cf : cfs) {
1563        Path src = cf.getPath();
1564        Path dst =  new Path(sidelineRegionDir, src.getName());
1565        if (fs.isFile(src)) {
1566          // simple file
1567          success = fs.rename(src, dst);
1568          if (!success) {
1569            String msg = "Unable to rename file " + src +  " to " + dst;
1570            LOG.error(msg);
1571            throw new IOException(msg);
1572          }
1573          continue;
1574        }
1575
1576        // is a directory.
1577        fs.mkdirs(dst);
1578
1579        LOG.info("Sidelining files from " + src + " into containing region " + dst);
1580        // FileSystem.rename is inconsistent with directories -- if the
1581        // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1582        // it moves the src into the dst dir resulting in (foo/a/b).  If
1583        // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1584        FileStatus[] hfiles = fs.listStatus(src);
1585        if (hfiles != null && hfiles.length > 0) {
1586          for (FileStatus hfile : hfiles) {
1587            success = fs.rename(hfile.getPath(), dst);
1588            if (!success) {
1589              String msg = "Unable to rename file " + src +  " to " + dst;
1590              LOG.error(msg);
1591              throw new IOException(msg);
1592            }
1593          }
1594        }
1595        LOG.debug("Sideline directory contents:");
1596        debugLsr(sidelineRegionDir);
1597      }
1598    }
1599
1600    LOG.info("Removing old region dir: " + regionDir);
1601    success = fs.delete(regionDir, true);
1602    if (!success) {
1603      String msg = "Unable to delete dir " + regionDir;
1604      LOG.error(msg);
1605      throw new IOException(msg);
1606    }
1607    return sidelineRegionDir;
1608  }
1609
1610  /**
1611   * Load the list of disabled tables in ZK into local set.
1612   * @throws ZooKeeperConnectionException
1613   * @throws IOException
1614   */
1615  private void loadTableStates()
1616  throws IOException {
1617    tableStates = MetaTableAccessor.getTableStates(connection);
1618    // Add hbase:meta so this tool keeps working. In hbase2, meta is always enabled though it
1619    // has no entry in the table states. HBCK doesn't work right w/ hbase2 but just do this in
1620    // meantime.
1621    this.tableStates.put(TableName.META_TABLE_NAME,
1622        new TableState(TableName.META_TABLE_NAME, TableState.State.ENABLED));
1623  }
1624
1625  /**
1626   * Check if the specified region's table is disabled.
1627   * @param tableName table to check status of
1628   */
1629  boolean isTableDisabled(TableName tableName) {
1630    return tableStates.containsKey(tableName)
1631        && tableStates.get(tableName)
1632        .inStates(TableState.State.DISABLED, TableState.State.DISABLING);
1633  }
1634
1635  /**
1636   * Scan HDFS for all regions, recording their information into
1637   * regionInfoMap
1638   */
1639  public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1640    Path rootDir = FSUtils.getRootDir(getConf());
1641    FileSystem fs = rootDir.getFileSystem(getConf());
1642
1643    // list all tables from HDFS
1644    List<FileStatus> tableDirs = Lists.newArrayList();
1645
1646    boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1647
1648    List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1649    for (Path path : paths) {
1650      TableName tableName = FSUtils.getTableName(path);
1651       if ((!checkMetaOnly &&
1652           isTableIncluded(tableName)) ||
1653           tableName.equals(TableName.META_TABLE_NAME)) {
1654         tableDirs.add(fs.getFileStatus(path));
1655       }
1656    }
1657
1658    // verify that version file exists
1659    if (!foundVersionFile) {
1660      errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1661          "Version file does not exist in root dir " + rootDir);
1662      if (shouldFixVersionFile()) {
1663        LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1664            + " file.");
1665        setShouldRerun();
1666        FSUtils.setVersion(fs, rootDir, getConf().getInt(
1667            HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1668            HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1669            HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1670      }
1671    }
1672
1673    // Avoid multithreading at table-level because already multithreaded internally at
1674    // region-level.  Additionally multithreading at table-level can lead to deadlock
1675    // if there are many tables in the cluster.  Since there are a limited # of threads
1676    // in the executor's thread pool and if we multithread at the table-level by putting
1677    // WorkItemHdfsDir callables into the executor, then we will have some threads in the
1678    // executor tied up solely in waiting for the tables' region-level calls to complete.
1679    // If there are enough tables then there will be no actual threads in the pool left
1680    // for the region-level callables to be serviced.
1681    for (FileStatus tableDir : tableDirs) {
1682      LOG.debug("Loading region dirs from " +tableDir.getPath());
1683      WorkItemHdfsDir item = new WorkItemHdfsDir(fs, errors, tableDir);
1684      try {
1685        item.call();
1686      } catch (ExecutionException e) {
1687        LOG.warn("Could not completely load table dir " +
1688            tableDir.getPath(), e.getCause());
1689      }
1690    }
1691    errors.print("");
1692  }
1693
1694  /**
1695   * Record the location of the hbase:meta region as found in ZooKeeper.
1696   */
1697  private boolean recordMetaRegion() throws IOException {
1698    RegionLocations rl = connection.locateRegion(TableName.META_TABLE_NAME,
1699        HConstants.EMPTY_START_ROW, false, false);
1700    if (rl == null) {
1701      errors.reportError(ERROR_CODE.NULL_META_REGION,
1702          "META region was not found in ZooKeeper");
1703      return false;
1704    }
1705    for (HRegionLocation metaLocation : rl.getRegionLocations()) {
1706      // Check if Meta region is valid and existing
1707      if (metaLocation == null ) {
1708        errors.reportError(ERROR_CODE.NULL_META_REGION,
1709            "META region location is null");
1710        return false;
1711      }
1712      if (metaLocation.getRegionInfo() == null) {
1713        errors.reportError(ERROR_CODE.NULL_META_REGION,
1714            "META location regionInfo is null");
1715        return false;
1716      }
1717      if (metaLocation.getHostname() == null) {
1718        errors.reportError(ERROR_CODE.NULL_META_REGION,
1719            "META location hostName is null");
1720        return false;
1721      }
1722      ServerName sn = metaLocation.getServerName();
1723      HbckRegionInfo.MetaEntry m = new HbckRegionInfo.MetaEntry(metaLocation.getRegion(), sn,
1724          EnvironmentEdgeManager.currentTime());
1725      HbckRegionInfo hbckRegionInfo = regionInfoMap.get(metaLocation.getRegion().getEncodedName());
1726      if (hbckRegionInfo == null) {
1727        regionInfoMap.put(metaLocation.getRegion().getEncodedName(), new HbckRegionInfo(m));
1728      } else {
1729        hbckRegionInfo.setMetaEntry(m);
1730      }
1731    }
1732    return true;
1733  }
1734
1735  private ZKWatcher createZooKeeperWatcher() throws IOException {
1736    return new ZKWatcher(getConf(), "hbase Fsck", new Abortable() {
1737      @Override
1738      public void abort(String why, Throwable e) {
1739        LOG.error(why, e);
1740        System.exit(1);
1741      }
1742
1743      @Override
1744      public boolean isAborted() {
1745        return false;
1746      }
1747
1748    });
1749  }
1750
1751  private ServerName getMetaRegionServerName(int replicaId)
1752  throws IOException, KeeperException {
1753    return new MetaTableLocator().getMetaRegionLocation(zkw, replicaId);
1754  }
1755
1756  /**
1757   * Contacts each regionserver and fetches metadata about regions.
1758   * @param regionServerList - the list of region servers to connect to
1759   * @throws IOException if a remote or network exception occurs
1760   */
1761  void processRegionServers(Collection<ServerName> regionServerList)
1762    throws IOException, InterruptedException {
1763
1764    List<WorkItemRegion> workItems = new ArrayList<>(regionServerList.size());
1765    List<Future<Void>> workFutures;
1766
1767    // loop to contact each region server in parallel
1768    for (ServerName rsinfo: regionServerList) {
1769      workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1770    }
1771
1772    workFutures = executor.invokeAll(workItems);
1773
1774    for(int i=0; i<workFutures.size(); i++) {
1775      WorkItemRegion item = workItems.get(i);
1776      Future<Void> f = workFutures.get(i);
1777      try {
1778        f.get();
1779      } catch(ExecutionException e) {
1780        LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1781            e.getCause());
1782      }
1783    }
1784  }
1785
1786  /**
1787   * Check consistency of all regions that have been found in previous phases.
1788   */
1789  private void checkAndFixConsistency()
1790  throws IOException, KeeperException, InterruptedException {
1791    // Divide the checks in two phases. One for default/primary replicas and another
1792    // for the non-primary ones. Keeps code cleaner this way.
1793
1794    List<CheckRegionConsistencyWorkItem> workItems = new ArrayList<>(regionInfoMap.size());
1795    for (java.util.Map.Entry<String, HbckRegionInfo> e: regionInfoMap.entrySet()) {
1796      if (e.getValue().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
1797        workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1798      }
1799    }
1800    checkRegionConsistencyConcurrently(workItems);
1801
1802    boolean prevHdfsCheck = shouldCheckHdfs();
1803    setCheckHdfs(false); //replicas don't have any hdfs data
1804    // Run a pass over the replicas and fix any assignment issues that exist on the currently
1805    // deployed/undeployed replicas.
1806    List<CheckRegionConsistencyWorkItem> replicaWorkItems = new ArrayList<>(regionInfoMap.size());
1807    for (java.util.Map.Entry<String, HbckRegionInfo> e: regionInfoMap.entrySet()) {
1808      if (e.getValue().getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
1809        replicaWorkItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1810      }
1811    }
1812    checkRegionConsistencyConcurrently(replicaWorkItems);
1813    setCheckHdfs(prevHdfsCheck);
1814
1815    // If some regions is skipped during checkRegionConsistencyConcurrently() phase, we might
1816    // not get accurate state of the hbase if continuing. The config here allows users to tune
1817    // the tolerance of number of skipped region.
1818    // TODO: evaluate the consequence to continue the hbck operation without config.
1819    int terminateThreshold =  getConf().getInt("hbase.hbck.skipped.regions.limit", 0);
1820    int numOfSkippedRegions = skippedRegions.size();
1821    if (numOfSkippedRegions > 0 && numOfSkippedRegions > terminateThreshold) {
1822      throw new IOException(numOfSkippedRegions
1823        + " region(s) could not be checked or repaired.  See logs for detail.");
1824    }
1825
1826    if (shouldCheckHdfs()) {
1827      checkAndFixTableStates();
1828    }
1829  }
1830
1831  /**
1832   * Check consistency of all regions using mulitple threads concurrently.
1833   */
1834  private void checkRegionConsistencyConcurrently(
1835    final List<CheckRegionConsistencyWorkItem> workItems)
1836    throws IOException, KeeperException, InterruptedException {
1837    if (workItems.isEmpty()) {
1838      return;  // nothing to check
1839    }
1840
1841    List<Future<Void>> workFutures = executor.invokeAll(workItems);
1842    for(Future<Void> f: workFutures) {
1843      try {
1844        f.get();
1845      } catch(ExecutionException e1) {
1846        LOG.warn("Could not check region consistency " , e1.getCause());
1847        if (e1.getCause() instanceof IOException) {
1848          throw (IOException)e1.getCause();
1849        } else if (e1.getCause() instanceof KeeperException) {
1850          throw (KeeperException)e1.getCause();
1851        } else if (e1.getCause() instanceof InterruptedException) {
1852          throw (InterruptedException)e1.getCause();
1853        } else {
1854          throw new IOException(e1.getCause());
1855        }
1856      }
1857    }
1858  }
1859
1860  class CheckRegionConsistencyWorkItem implements Callable<Void> {
1861    private final String key;
1862    private final HbckRegionInfo hbi;
1863
1864    CheckRegionConsistencyWorkItem(String key, HbckRegionInfo hbi) {
1865      this.key = key;
1866      this.hbi = hbi;
1867    }
1868
1869    @Override
1870    public synchronized Void call() throws Exception {
1871      try {
1872        checkRegionConsistency(key, hbi);
1873      } catch (Exception e) {
1874        // If the region is non-META region, skip this region and send warning/error message; if
1875        // the region is META region, we should not continue.
1876        LOG.warn("Unable to complete check or repair the region '" + hbi.getRegionNameAsString()
1877          + "'.", e);
1878        if (hbi.getHdfsHRI().isMetaRegion()) {
1879          throw e;
1880        }
1881        LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'");
1882        addSkippedRegion(hbi);
1883      }
1884      return null;
1885    }
1886  }
1887
1888  private void addSkippedRegion(final HbckRegionInfo hbi) {
1889    Set<String> skippedRegionNames = skippedRegions.get(hbi.getTableName());
1890    if (skippedRegionNames == null) {
1891      skippedRegionNames = new HashSet<>();
1892    }
1893    skippedRegionNames.add(hbi.getRegionNameAsString());
1894    skippedRegions.put(hbi.getTableName(), skippedRegionNames);
1895  }
1896
1897  /**
1898   * Check and fix table states, assumes full info available:
1899   * - tableInfos
1900   * - empty tables loaded
1901   */
1902  private void checkAndFixTableStates() throws IOException {
1903    // first check dangling states
1904    for (Entry<TableName, TableState> entry : tableStates.entrySet()) {
1905      TableName tableName = entry.getKey();
1906      TableState tableState = entry.getValue();
1907      HbckTableInfo tableInfo = tablesInfo.get(tableName);
1908      if (isTableIncluded(tableName)
1909          && !tableName.isSystemTable()
1910          && tableInfo == null) {
1911        if (fixMeta) {
1912          MetaTableAccessor.deleteTableState(connection, tableName);
1913          TableState state = MetaTableAccessor.getTableState(connection, tableName);
1914          if (state != null) {
1915            errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,
1916                tableName + " unable to delete dangling table state " + tableState);
1917          }
1918        } else if (!checkMetaOnly) {
1919          // dangling table state in meta if checkMetaOnly is false. If checkMetaOnly is
1920          // true, tableInfo will be null as tablesInfo are not polulated for all tables from hdfs
1921          errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,
1922              tableName + " has dangling table state " + tableState);
1923        }
1924      }
1925    }
1926    // check that all tables have states
1927    for (TableName tableName : tablesInfo.keySet()) {
1928      if (isTableIncluded(tableName) && !tableStates.containsKey(tableName)) {
1929        if (fixMeta) {
1930          MetaTableAccessor.updateTableState(connection, tableName, TableState.State.ENABLED);
1931          TableState newState = MetaTableAccessor.getTableState(connection, tableName);
1932          if (newState == null) {
1933            errors.reportError(ERROR_CODE.NO_TABLE_STATE,
1934                "Unable to change state for table " + tableName + " in meta ");
1935          }
1936        } else {
1937          errors.reportError(ERROR_CODE.NO_TABLE_STATE,
1938              tableName + " has no state in meta ");
1939        }
1940      }
1941    }
1942  }
1943
1944  private void preCheckPermission() throws IOException, AccessDeniedException {
1945    if (shouldIgnorePreCheckPermission()) {
1946      return;
1947    }
1948
1949    Path hbaseDir = FSUtils.getRootDir(getConf());
1950    FileSystem fs = hbaseDir.getFileSystem(getConf());
1951    UserProvider userProvider = UserProvider.instantiate(getConf());
1952    UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1953    FileStatus[] files = fs.listStatus(hbaseDir);
1954    for (FileStatus file : files) {
1955      try {
1956        FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1957      } catch (AccessDeniedException ace) {
1958        LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1959        errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1960          + " does not have write perms to " + file.getPath()
1961          + ". Please rerun hbck as hdfs user " + file.getOwner());
1962        throw ace;
1963      }
1964    }
1965  }
1966
1967  /**
1968   * Deletes region from meta table
1969   */
1970  private void deleteMetaRegion(HbckRegionInfo hi) throws IOException {
1971    deleteMetaRegion(hi.getMetaEntry().getRegionName());
1972  }
1973
1974  /**
1975   * Deletes region from meta table
1976   */
1977  private void deleteMetaRegion(byte[] metaKey) throws IOException {
1978    Delete d = new Delete(metaKey);
1979    meta.delete(d);
1980    LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1981  }
1982
1983  /**
1984   * Reset the split parent region info in meta table
1985   */
1986  private void resetSplitParent(HbckRegionInfo hi) throws IOException {
1987    RowMutations mutations = new RowMutations(hi.getMetaEntry().getRegionName());
1988    Delete d = new Delete(hi.getMetaEntry().getRegionName());
1989    d.addColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1990    d.addColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1991    mutations.add(d);
1992
1993    RegionInfo hri = RegionInfoBuilder.newBuilder(hi.getMetaEntry())
1994        .setOffline(false)
1995        .setSplit(false)
1996        .build();
1997    Put p = MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
1998    mutations.add(p);
1999
2000    meta.mutateRow(mutations);
2001    LOG.info("Reset split parent " + hi.getMetaEntry().getRegionNameAsString() + " in META");
2002  }
2003
2004  /**
2005   * This backwards-compatibility wrapper for permanently offlining a region
2006   * that should not be alive.  If the region server does not support the
2007   * "offline" method, it will use the closest unassign method instead.  This
2008   * will basically work until one attempts to disable or delete the affected
2009   * table.  The problem has to do with in-memory only master state, so
2010   * restarting the HMaster or failing over to another should fix this.
2011   */
2012  void offline(byte[] regionName) throws IOException {
2013    String regionString = Bytes.toStringBinary(regionName);
2014    if (!rsSupportsOffline) {
2015      LOG.warn(
2016          "Using unassign region " + regionString + " instead of using offline method, you should" +
2017              " restart HMaster after these repairs");
2018      admin.unassign(regionName, true);
2019      return;
2020    }
2021
2022    // first time we assume the rs's supports #offline.
2023    try {
2024      LOG.info("Offlining region " + regionString);
2025      admin.offline(regionName);
2026    } catch (IOException ioe) {
2027      String notFoundMsg = "java.lang.NoSuchMethodException: " +
2028          "org.apache.hadoop.hbase.master.HMaster.offline([B)";
2029      if (ioe.getMessage().contains(notFoundMsg)) {
2030        LOG.warn("Using unassign region " + regionString +
2031            " instead of using offline method, you should" +
2032            " restart HMaster after these repairs");
2033        rsSupportsOffline = false; // in the future just use unassign
2034        admin.unassign(regionName, true);
2035        return;
2036      }
2037      throw ioe;
2038    }
2039  }
2040
2041  /**
2042   * Attempts to undeploy a region from a region server based in information in
2043   * META.  Any operations that modify the file system should make sure that
2044   * its corresponding region is not deployed to prevent data races.
2045   *
2046   * A separate call is required to update the master in-memory region state
2047   * kept in the AssignementManager.  Because disable uses this state instead of
2048   * that found in META, we can't seem to cleanly disable/delete tables that
2049   * have been hbck fixed.  When used on a version of HBase that does not have
2050   * the offline ipc call exposed on the master (&lt;0.90.5, &lt;0.92.0) a master
2051   * restart or failover may be required.
2052   */
2053  void closeRegion(HbckRegionInfo hi) throws IOException, InterruptedException {
2054    if (hi.getMetaEntry() == null && hi.getHdfsEntry() == null) {
2055      undeployRegions(hi);
2056      return;
2057    }
2058
2059    // get assignment info and hregioninfo from meta.
2060    Get get = new Get(hi.getRegionName());
2061    get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2062    get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
2063    get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
2064    // also get the locations of the replicas to close if the primary region is being closed
2065    if (hi.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
2066      int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2067      for (int i = 0; i < numReplicas; i++) {
2068        get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
2069        get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
2070      }
2071    }
2072    Result r = meta.get(get);
2073    RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
2074    if (rl == null) {
2075      LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
2076          " since meta does not have handle to reach it");
2077      return;
2078    }
2079    for (HRegionLocation h : rl.getRegionLocations()) {
2080      ServerName serverName = h.getServerName();
2081      if (serverName == null) {
2082        errors.reportError("Unable to close region "
2083            + hi.getRegionNameAsString() +  " because meta does not "
2084            + "have handle to reach it.");
2085        continue;
2086      }
2087      RegionInfo hri = h.getRegionInfo();
2088      if (hri == null) {
2089        LOG.warn("Unable to close region " + hi.getRegionNameAsString()
2090            + " because hbase:meta had invalid or missing "
2091            + HConstants.CATALOG_FAMILY_STR + ":"
2092            + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
2093            + " qualifier value.");
2094        continue;
2095      }
2096      // close the region -- close files and remove assignment
2097      HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
2098    }
2099  }
2100
2101  private void undeployRegions(HbckRegionInfo hi) throws IOException, InterruptedException {
2102    undeployRegionsForHbi(hi);
2103    // undeploy replicas of the region (but only if the method is invoked for the primary)
2104    if (hi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
2105      return;
2106    }
2107    int numReplicas = admin.getDescriptor(hi.getTableName()).getRegionReplication();
2108    for (int i = 1; i < numReplicas; i++) {
2109      if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
2110      RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
2111          hi.getPrimaryHRIForDeployedReplica(), i);
2112      HbckRegionInfo h = regionInfoMap.get(hri.getEncodedName());
2113      if (h != null) {
2114        undeployRegionsForHbi(h);
2115        //set skip checks; we undeployed it, and we don't want to evaluate this anymore
2116        //in consistency checks
2117        h.setSkipChecks(true);
2118      }
2119    }
2120  }
2121
2122  private void undeployRegionsForHbi(HbckRegionInfo hi) throws IOException, InterruptedException {
2123    for (HbckRegionInfo.OnlineEntry rse : hi.getOnlineEntries()) {
2124      LOG.debug("Undeploy region "  + rse.getRegionInfo() + " from " + rse.getServerName());
2125      try {
2126        HBaseFsckRepair
2127            .closeRegionSilentlyAndWait(connection, rse.getServerName(), rse.getRegionInfo());
2128        offline(rse.getRegionInfo().getRegionName());
2129      } catch (IOException ioe) {
2130        LOG.warn("Got exception when attempting to offline region "
2131            + Bytes.toString(rse.getRegionInfo().getRegionName()), ioe);
2132      }
2133    }
2134  }
2135
2136  private void tryAssignmentRepair(HbckRegionInfo hbi, String msg) throws IOException,
2137    KeeperException, InterruptedException {
2138    // If we are trying to fix the errors
2139    if (shouldFixAssignments()) {
2140      errors.print(msg);
2141      undeployRegions(hbi);
2142      setShouldRerun();
2143      RegionInfo hri = hbi.getHdfsHRI();
2144      if (hri == null) {
2145        hri = hbi.getMetaEntry();
2146      }
2147      HBaseFsckRepair.fixUnassigned(admin, hri);
2148      HBaseFsckRepair.waitUntilAssigned(admin, hri);
2149
2150      // also assign replicas if needed (do it only when this call operates on a primary replica)
2151      if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) return;
2152      int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
2153      for (int i = 1; i < replicationCount; i++) {
2154        hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
2155        HbckRegionInfo h = regionInfoMap.get(hri.getEncodedName());
2156        if (h != null) {
2157          undeployRegions(h);
2158          //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore
2159          //in consistency checks
2160          h.setSkipChecks(true);
2161        }
2162        HBaseFsckRepair.fixUnassigned(admin, hri);
2163        HBaseFsckRepair.waitUntilAssigned(admin, hri);
2164      }
2165
2166    }
2167  }
2168
2169  /**
2170   * Check a single region for consistency and correct deployment.
2171   */
2172  private void checkRegionConsistency(final String key, final HbckRegionInfo hbi)
2173      throws IOException, KeeperException, InterruptedException {
2174
2175    if (hbi.isSkipChecks()) return;
2176    String descriptiveName = hbi.toString();
2177    boolean inMeta = hbi.getMetaEntry() != null;
2178    // In case not checking HDFS, assume the region is on HDFS
2179    boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
2180    boolean hasMetaAssignment = inMeta && hbi.getMetaEntry().regionServer != null;
2181    boolean isDeployed = !hbi.getDeployedOn().isEmpty();
2182    boolean isMultiplyDeployed = hbi.getDeployedOn().size() > 1;
2183    boolean deploymentMatchesMeta =
2184      hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
2185      hbi.getMetaEntry().regionServer.equals(hbi.getDeployedOn().get(0));
2186    boolean splitParent =
2187        inMeta && hbi.getMetaEntry().isSplit() && hbi.getMetaEntry().isOffline();
2188    boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.getMetaEntry().getTable());
2189    boolean recentlyModified = inHdfs &&
2190      hbi.getModTime() + timelag > EnvironmentEdgeManager.currentTime();
2191
2192    // ========== First the healthy cases =============
2193    if (hbi.containsOnlyHdfsEdits()) {
2194      return;
2195    }
2196    if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
2197      return;
2198    } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
2199      LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
2200        "tabled that is not deployed");
2201      return;
2202    } else if (recentlyModified) {
2203      LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
2204      return;
2205    }
2206    // ========== Cases where the region is not in hbase:meta =============
2207    else if (!inMeta && !inHdfs && !isDeployed) {
2208      // We shouldn't have record of this region at all then!
2209      assert false : "Entry for region with no data";
2210    } else if (!inMeta && !inHdfs && isDeployed) {
2211      errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
2212          + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
2213          "deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));
2214      if (shouldFixAssignments()) {
2215        undeployRegions(hbi);
2216      }
2217
2218    } else if (!inMeta && inHdfs && !isDeployed) {
2219      if (hbi.isMerged()) {
2220        // This region has already been merged, the remaining hdfs file will be
2221        // cleaned by CatalogJanitor later
2222        hbi.setSkipChecks(true);
2223        LOG.info("Region " + descriptiveName
2224            + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
2225        return;
2226      }
2227      errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
2228          + descriptiveName + " on HDFS, but not listed in hbase:meta " +
2229          "or deployed on any region server");
2230      // restore region consistency of an adopted orphan
2231      if (shouldFixMeta()) {
2232        if (!hbi.isHdfsRegioninfoPresent()) {
2233          LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
2234              +  " in table integrity repair phase if -fixHdfsOrphans was" +
2235              " used.");
2236          return;
2237        }
2238
2239        RegionInfo hri = hbi.getHdfsHRI();
2240        HbckTableInfo tableInfo = tablesInfo.get(hri.getTable());
2241
2242        for (RegionInfo region : tableInfo.getRegionsFromMeta(this.regionInfoMap)) {
2243          if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0
2244              && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(),
2245                hri.getEndKey()) >= 0)
2246              && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) {
2247            if(region.isSplit() || region.isOffline()) continue;
2248            Path regionDir = hbi.getHdfsRegionDir();
2249            FileSystem fs = regionDir.getFileSystem(getConf());
2250            List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir);
2251            for (Path familyDir : familyDirs) {
2252              List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);
2253              for (Path referenceFilePath : referenceFilePaths) {
2254                Path parentRegionDir =
2255                    StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();
2256                if (parentRegionDir.toString().endsWith(region.getEncodedName())) {
2257                  LOG.warn(hri + " start and stop keys are in the range of " + region
2258                      + ". The region might not be cleaned up from hdfs when region " + region
2259                      + " split failed. Hence deleting from hdfs.");
2260                  HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
2261                    regionDir.getParent(), hri);
2262                  return;
2263                }
2264              }
2265            }
2266          }
2267        }
2268        LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
2269        int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2270        HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2271            admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
2272              .getLiveServerMetrics().keySet(), numReplicas);
2273
2274        tryAssignmentRepair(hbi, "Trying to reassign region...");
2275      }
2276
2277    } else if (!inMeta && inHdfs && isDeployed) {
2278      errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
2279          + " not in META, but deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));
2280      debugLsr(hbi.getHdfsRegionDir());
2281      if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
2282        // for replicas, this means that we should undeploy the region (we would have
2283        // gone over the primaries and fixed meta holes in first phase under
2284        // checkAndFixConsistency; we shouldn't get the condition !inMeta at
2285        // this stage unless unwanted replica)
2286        if (shouldFixAssignments()) {
2287          undeployRegionsForHbi(hbi);
2288        }
2289      }
2290      if (shouldFixMeta() && hbi.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
2291        if (!hbi.isHdfsRegioninfoPresent()) {
2292          LOG.error("This should have been repaired in table integrity repair phase");
2293          return;
2294        }
2295
2296        LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
2297        int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2298        HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2299            admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
2300              .getLiveServerMetrics().keySet(), numReplicas);
2301        tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2302      }
2303
2304    // ========== Cases where the region is in hbase:meta =============
2305    } else if (inMeta && inHdfs && !isDeployed && splitParent) {
2306      // check whether this is an actual error, or just transient state where parent
2307      // is not cleaned
2308      if (hbi.getMetaEntry().splitA != null && hbi.getMetaEntry().splitB != null) {
2309        // check that split daughters are there
2310        HbckRegionInfo infoA = this.regionInfoMap.get(hbi.getMetaEntry().splitA.getEncodedName());
2311        HbckRegionInfo infoB = this.regionInfoMap.get(hbi.getMetaEntry().splitB.getEncodedName());
2312        if (infoA != null && infoB != null) {
2313          // we already processed or will process daughters. Move on, nothing to see here.
2314          hbi.setSkipChecks(true);
2315          return;
2316        }
2317      }
2318
2319      // For Replica region, we need to do a similar check. If replica is not split successfully,
2320      // error is going to be reported against primary daughter region.
2321      if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
2322        LOG.info("Region " + descriptiveName + " is a split parent in META, in HDFS, "
2323            + "and not deployed on any region server. This may be transient.");
2324        hbi.setSkipChecks(true);
2325        return;
2326      }
2327
2328      errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2329          + descriptiveName + " is a split parent in META, in HDFS, "
2330          + "and not deployed on any region server. This could be transient, "
2331          + "consider to run the catalog janitor first!");
2332      if (shouldFixSplitParents()) {
2333        setShouldRerun();
2334        resetSplitParent(hbi);
2335      }
2336    } else if (inMeta && !inHdfs && !isDeployed) {
2337      errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2338          + descriptiveName + " found in META, but not in HDFS "
2339          + "or deployed on any region server.");
2340      if (shouldFixMeta()) {
2341        deleteMetaRegion(hbi);
2342      }
2343    } else if (inMeta && !inHdfs && isDeployed) {
2344      errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2345          + " found in META, but not in HDFS, " +
2346          "and deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));
2347      // We treat HDFS as ground truth.  Any information in meta is transient
2348      // and equivalent data can be regenerated.  So, lets unassign and remove
2349      // these problems from META.
2350      if (shouldFixAssignments()) {
2351        errors.print("Trying to fix unassigned region...");
2352        undeployRegions(hbi);
2353      }
2354      if (shouldFixMeta()) {
2355        // wait for it to complete
2356        deleteMetaRegion(hbi);
2357      }
2358    } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2359      errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2360          + " not deployed on any region server.");
2361      tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2362    } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2363      errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2364          "Region " + descriptiveName + " should not be deployed according " +
2365          "to META, but is deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));
2366      if (shouldFixAssignments()) {
2367        errors.print("Trying to close the region " + descriptiveName);
2368        setShouldRerun();
2369        HBaseFsckRepair.fixMultiAssignment(connection, hbi.getMetaEntry(), hbi.getDeployedOn());
2370      }
2371    } else if (inMeta && inHdfs && isMultiplyDeployed) {
2372      errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2373          + " is listed in hbase:meta on region server " + hbi.getMetaEntry().regionServer
2374          + " but is multiply assigned to region servers " +
2375          Joiner.on(", ").join(hbi.getDeployedOn()));
2376      // If we are trying to fix the errors
2377      if (shouldFixAssignments()) {
2378        errors.print("Trying to fix assignment error...");
2379        setShouldRerun();
2380        HBaseFsckRepair.fixMultiAssignment(connection, hbi.getMetaEntry(), hbi.getDeployedOn());
2381      }
2382    } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2383      errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2384          + descriptiveName + " listed in hbase:meta on region server " +
2385          hbi.getMetaEntry().regionServer + " but found on region server " +
2386          hbi.getDeployedOn().get(0));
2387      // If we are trying to fix the errors
2388      if (shouldFixAssignments()) {
2389        errors.print("Trying to fix assignment error...");
2390        setShouldRerun();
2391        HBaseFsckRepair.fixMultiAssignment(connection, hbi.getMetaEntry(), hbi.getDeployedOn());
2392        HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2393      }
2394    } else {
2395      errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2396          " is in an unforeseen state:" +
2397          " inMeta=" + inMeta +
2398          " inHdfs=" + inHdfs +
2399          " isDeployed=" + isDeployed +
2400          " isMultiplyDeployed=" + isMultiplyDeployed +
2401          " deploymentMatchesMeta=" + deploymentMatchesMeta +
2402          " shouldBeDeployed=" + shouldBeDeployed);
2403    }
2404  }
2405
2406  /**
2407   * Checks tables integrity. Goes over all regions and scans the tables.
2408   * Collects all the pieces for each table and checks if there are missing,
2409   * repeated or overlapping ones.
2410   * @throws IOException
2411   */
2412  SortedMap<TableName, HbckTableInfo> checkIntegrity() throws IOException {
2413    tablesInfo = new TreeMap<>();
2414    LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2415    for (HbckRegionInfo hbi : regionInfoMap.values()) {
2416      // Check only valid, working regions
2417      if (hbi.getMetaEntry() == null) {
2418        // this assumes that consistency check has run loadMetaEntry
2419        Path p = hbi.getHdfsRegionDir();
2420        if (p == null) {
2421          errors.report("No regioninfo in Meta or HDFS. " + hbi);
2422        }
2423
2424        // TODO test.
2425        continue;
2426      }
2427      if (hbi.getMetaEntry().regionServer == null) {
2428        errors.detail("Skipping region because no region server: " + hbi);
2429        continue;
2430      }
2431      if (hbi.getMetaEntry().isOffline()) {
2432        errors.detail("Skipping region because it is offline: " + hbi);
2433        continue;
2434      }
2435      if (hbi.containsOnlyHdfsEdits()) {
2436        errors.detail("Skipping region because it only contains edits" + hbi);
2437        continue;
2438      }
2439
2440      // Missing regionDir or over-deployment is checked elsewhere. Include
2441      // these cases in modTInfo, so we can evaluate those regions as part of
2442      // the region chain in META
2443      //if (hbi.foundRegionDir == null) continue;
2444      //if (hbi.deployedOn.size() != 1) continue;
2445      if (hbi.getDeployedOn().isEmpty()) {
2446        continue;
2447      }
2448
2449      // We should be safe here
2450      TableName tableName = hbi.getMetaEntry().getTable();
2451      HbckTableInfo modTInfo = tablesInfo.get(tableName);
2452      if (modTInfo == null) {
2453        modTInfo = new HbckTableInfo(tableName, this);
2454      }
2455      for (ServerName server : hbi.getDeployedOn()) {
2456        modTInfo.addServer(server);
2457      }
2458
2459      if (!hbi.isSkipChecks()) {
2460        modTInfo.addRegionInfo(hbi);
2461      }
2462
2463      tablesInfo.put(tableName, modTInfo);
2464    }
2465
2466    loadTableInfosForTablesWithNoRegion();
2467
2468    logParallelMerge();
2469    for (HbckTableInfo tInfo : tablesInfo.values()) {
2470      TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2471      if (!tInfo.checkRegionChain(handler)) {
2472        errors.report("Found inconsistency in table " + tInfo.getName());
2473      }
2474    }
2475    return tablesInfo;
2476  }
2477
2478  /** Loads table info's for tables that may not have been included, since there are no
2479   * regions reported for the table, but table dir is there in hdfs
2480   */
2481  private void loadTableInfosForTablesWithNoRegion() throws IOException {
2482    Map<String, TableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2483    for (TableDescriptor htd : allTables.values()) {
2484      if (checkMetaOnly && !htd.isMetaTable()) {
2485        continue;
2486      }
2487
2488      TableName tableName = htd.getTableName();
2489      if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2490        HbckTableInfo tableInfo = new HbckTableInfo(tableName, this);
2491        tableInfo.htds.add(htd);
2492        tablesInfo.put(htd.getTableName(), tableInfo);
2493      }
2494    }
2495  }
2496
2497  /**
2498   * Merge hdfs data by moving from contained HbckRegionInfo into targetRegionDir.
2499   * @return number of file move fixes done to merge regions.
2500   */
2501  public int mergeRegionDirs(Path targetRegionDir, HbckRegionInfo contained) throws IOException {
2502    int fileMoves = 0;
2503    String thread = Thread.currentThread().getName();
2504    LOG.debug("[" + thread + "] Contained region dir after close and pause");
2505    debugLsr(contained.getHdfsRegionDir());
2506
2507    // rename the contained into the container.
2508    FileSystem fs = targetRegionDir.getFileSystem(getConf());
2509    FileStatus[] dirs = null;
2510    try {
2511      dirs = fs.listStatus(contained.getHdfsRegionDir());
2512    } catch (FileNotFoundException fnfe) {
2513      // region we are attempting to merge in is not present!  Since this is a merge, there is
2514      // no harm skipping this region if it does not exist.
2515      if (!fs.exists(contained.getHdfsRegionDir())) {
2516        LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2517            + " is missing. Assuming already sidelined or moved.");
2518      } else {
2519        sidelineRegionDir(fs, contained);
2520      }
2521      return fileMoves;
2522    }
2523
2524    if (dirs == null) {
2525      if (!fs.exists(contained.getHdfsRegionDir())) {
2526        LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2527            + " already sidelined.");
2528      } else {
2529        sidelineRegionDir(fs, contained);
2530      }
2531      return fileMoves;
2532    }
2533
2534    for (FileStatus cf : dirs) {
2535      Path src = cf.getPath();
2536      Path dst =  new Path(targetRegionDir, src.getName());
2537
2538      if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2539        // do not copy the old .regioninfo file.
2540        continue;
2541      }
2542
2543      if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2544        // do not copy the .oldlogs files
2545        continue;
2546      }
2547
2548      LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2549      // FileSystem.rename is inconsistent with directories -- if the
2550      // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2551      // it moves the src into the dst dir resulting in (foo/a/b).  If
2552      // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2553      for (FileStatus hfile : fs.listStatus(src)) {
2554        boolean success = fs.rename(hfile.getPath(), dst);
2555        if (success) {
2556          fileMoves++;
2557        }
2558      }
2559      LOG.debug("[" + thread + "] Sideline directory contents:");
2560      debugLsr(targetRegionDir);
2561    }
2562
2563    // if all success.
2564    sidelineRegionDir(fs, contained);
2565    LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2566        getSidelineDir());
2567    debugLsr(contained.getHdfsRegionDir());
2568
2569    return fileMoves;
2570  }
2571
2572
2573  static class WorkItemOverlapMerge implements Callable<Void> {
2574    private TableIntegrityErrorHandler handler;
2575    Collection<HbckRegionInfo> overlapgroup;
2576
2577    WorkItemOverlapMerge(Collection<HbckRegionInfo> overlapgroup,
2578        TableIntegrityErrorHandler handler) {
2579      this.handler = handler;
2580      this.overlapgroup = overlapgroup;
2581    }
2582
2583    @Override
2584    public Void call() throws Exception {
2585      handler.handleOverlapGroup(overlapgroup);
2586      return null;
2587    }
2588  };
2589
2590  /**
2591   * Return a list of user-space table names whose metadata have not been
2592   * modified in the last few milliseconds specified by timelag
2593   * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
2594   * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
2595   * milliseconds specified by timelag, then the table is a candidate to be returned.
2596   * @return tables that have not been modified recently
2597   * @throws IOException if an error is encountered
2598   */
2599  TableDescriptor[] getTables(AtomicInteger numSkipped) {
2600    List<TableName> tableNames = new ArrayList<>();
2601    long now = EnvironmentEdgeManager.currentTime();
2602
2603    for (HbckRegionInfo hbi : regionInfoMap.values()) {
2604      HbckRegionInfo.MetaEntry info = hbi.getMetaEntry();
2605
2606      // if the start key is zero, then we have found the first region of a table.
2607      // pick only those tables that were not modified in the last few milliseconds.
2608      if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
2609        if (info.modTime + timelag < now) {
2610          tableNames.add(info.getTable());
2611        } else {
2612          numSkipped.incrementAndGet(); // one more in-flux table
2613        }
2614      }
2615    }
2616    return getTableDescriptors(tableNames);
2617  }
2618
2619  TableDescriptor[] getTableDescriptors(List<TableName> tableNames) {
2620      LOG.info("getTableDescriptors == tableNames => " + tableNames);
2621    try (Connection conn = ConnectionFactory.createConnection(getConf());
2622        Admin admin = conn.getAdmin()) {
2623      List<TableDescriptor> tds = admin.listTableDescriptors(tableNames);
2624      return tds.toArray(new TableDescriptor[tds.size()]);
2625    } catch (IOException e) {
2626      LOG.debug("Exception getting table descriptors", e);
2627    }
2628    return new TableDescriptor[0];
2629  }
2630
2631  /**
2632   * Gets the entry in regionInfo corresponding to the the given encoded
2633   * region name. If the region has not been seen yet, a new entry is added
2634   * and returned.
2635   */
2636  private synchronized HbckRegionInfo getOrCreateInfo(String name) {
2637    HbckRegionInfo hbi = regionInfoMap.get(name);
2638    if (hbi == null) {
2639      hbi = new HbckRegionInfo(null);
2640      regionInfoMap.put(name, hbi);
2641    }
2642    return hbi;
2643  }
2644
2645  private void checkAndFixReplication() throws ReplicationException {
2646    ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, errors);
2647    checker.checkUnDeletedQueues();
2648
2649    if (checker.hasUnDeletedQueues() && this.fixReplication) {
2650      checker.fixUnDeletedQueues();
2651      setShouldRerun();
2652    }
2653  }
2654
2655  /**
2656    * Check values in regionInfo for hbase:meta
2657    * Check if zero or more than one regions with hbase:meta are found.
2658    * If there are inconsistencies (i.e. zero or more than one regions
2659    * pretend to be holding the hbase:meta) try to fix that and report an error.
2660    * @throws IOException from HBaseFsckRepair functions
2661    * @throws KeeperException
2662    * @throws InterruptedException
2663    */
2664  boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
2665    Map<Integer, HbckRegionInfo> metaRegions = new HashMap<>();
2666    for (HbckRegionInfo value : regionInfoMap.values()) {
2667      if (value.getMetaEntry() != null && value.getMetaEntry().isMetaRegion()) {
2668        metaRegions.put(value.getReplicaId(), value);
2669      }
2670    }
2671    int metaReplication = admin.getTableDescriptor(TableName.META_TABLE_NAME)
2672        .getRegionReplication();
2673    boolean noProblem = true;
2674    // There will be always entries in regionInfoMap corresponding to hbase:meta & its replicas
2675    // Check the deployed servers. It should be exactly one server for each replica.
2676    for (int i = 0; i < metaReplication; i++) {
2677      HbckRegionInfo metaHbckRegionInfo = metaRegions.remove(i);
2678      List<ServerName> servers = new ArrayList<>();
2679      if (metaHbckRegionInfo != null) {
2680        servers = metaHbckRegionInfo.getDeployedOn();
2681      }
2682      if (servers.size() != 1) {
2683        noProblem = false;
2684        if (servers.isEmpty()) {
2685          assignMetaReplica(i);
2686        } else if (servers.size() > 1) {
2687          errors
2688          .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId " +
2689                       metaHbckRegionInfo.getReplicaId() + " is found on more than one region.");
2690          if (shouldFixAssignments()) {
2691            errors.print("Trying to fix a problem with hbase:meta, replicaId " +
2692                metaHbckRegionInfo.getReplicaId() + "..");
2693            setShouldRerun();
2694            // try fix it (treat is a dupe assignment)
2695            HBaseFsckRepair
2696                .fixMultiAssignment(connection, metaHbckRegionInfo.getMetaEntry(), servers);
2697          }
2698        }
2699      }
2700    }
2701    // unassign whatever is remaining in metaRegions. They are excess replicas.
2702    for (Map.Entry<Integer, HbckRegionInfo> entry : metaRegions.entrySet()) {
2703      noProblem = false;
2704      errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2705          "hbase:meta replicas are deployed in excess. Configured " + metaReplication +
2706          ", deployed " + metaRegions.size());
2707      if (shouldFixAssignments()) {
2708        errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() +
2709            " of hbase:meta..");
2710        setShouldRerun();
2711        unassignMetaReplica(entry.getValue());
2712      }
2713    }
2714    // if noProblem is false, rerun hbck with hopefully fixed META
2715    // if noProblem is true, no errors, so continue normally
2716    return noProblem;
2717  }
2718
2719  private void unassignMetaReplica(HbckRegionInfo hi)
2720      throws IOException, InterruptedException, KeeperException {
2721    undeployRegions(hi);
2722    ZKUtil
2723        .deleteNode(zkw, zkw.getZNodePaths().getZNodeForReplica(hi.getMetaEntry().getReplicaId()));
2724  }
2725
2726  private void assignMetaReplica(int replicaId)
2727      throws IOException, KeeperException, InterruptedException {
2728    errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " +
2729        replicaId +" is not found on any region.");
2730    if (shouldFixAssignments()) {
2731      errors.print("Trying to fix a problem with hbase:meta..");
2732      setShouldRerun();
2733      // try to fix it (treat it as unassigned region)
2734      RegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
2735          RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId);
2736      HBaseFsckRepair.fixUnassigned(admin, h);
2737      HBaseFsckRepair.waitUntilAssigned(admin, h);
2738    }
2739  }
2740
2741  /**
2742   * Scan hbase:meta, adding all regions found to the regionInfo map.
2743   * @throws IOException if an error is encountered
2744   */
2745  boolean loadMetaEntries() throws IOException {
2746    MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
2747      int countRecord = 1;
2748
2749      // comparator to sort KeyValues with latest modtime
2750      final Comparator<Cell> comp = new Comparator<Cell>() {
2751        @Override
2752        public int compare(Cell k1, Cell k2) {
2753          return Long.compare(k1.getTimestamp(), k2.getTimestamp());
2754        }
2755      };
2756
2757      @Override
2758      public boolean visit(Result result) throws IOException {
2759        try {
2760
2761          // record the latest modification of this META record
2762          long ts =  Collections.max(result.listCells(), comp).getTimestamp();
2763          RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
2764          if (rl == null) {
2765            emptyRegionInfoQualifiers.add(result);
2766            errors.reportError(ERROR_CODE.EMPTY_META_CELL,
2767              "Empty REGIONINFO_QUALIFIER found in hbase:meta");
2768            return true;
2769          }
2770          ServerName sn = null;
2771          if (rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID) == null ||
2772              rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
2773            emptyRegionInfoQualifiers.add(result);
2774            errors.reportError(ERROR_CODE.EMPTY_META_CELL,
2775              "Empty REGIONINFO_QUALIFIER found in hbase:meta");
2776            return true;
2777          }
2778          RegionInfo hri = rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
2779          if (!(isTableIncluded(hri.getTable())
2780              || hri.isMetaRegion())) {
2781            return true;
2782          }
2783          PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(result);
2784          for (HRegionLocation h : rl.getRegionLocations()) {
2785            if (h == null || h.getRegionInfo() == null) {
2786              continue;
2787            }
2788            sn = h.getServerName();
2789            hri = h.getRegionInfo();
2790
2791            HbckRegionInfo.MetaEntry m = null;
2792            if (hri.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
2793              m = new HbckRegionInfo.MetaEntry(hri, sn, ts, daughters.getFirst(),
2794                  daughters.getSecond());
2795            } else {
2796              m = new HbckRegionInfo.MetaEntry(hri, sn, ts, null, null);
2797            }
2798            HbckRegionInfo previous = regionInfoMap.get(hri.getEncodedName());
2799            if (previous == null) {
2800              regionInfoMap.put(hri.getEncodedName(), new HbckRegionInfo(m));
2801            } else if (previous.getMetaEntry() == null) {
2802              previous.setMetaEntry(m);
2803            } else {
2804              throw new IOException("Two entries in hbase:meta are same " + previous);
2805            }
2806          }
2807          List<RegionInfo> mergeParents = MetaTableAccessor.getMergeRegions(result.rawCells());
2808          if (mergeParents != null) {
2809            for (RegionInfo mergeRegion : mergeParents) {
2810              if (mergeRegion != null) {
2811                // This region is already being merged
2812                HbckRegionInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
2813                hbInfo.setMerged(true);
2814              }
2815            }
2816          }
2817
2818          // show proof of progress to the user, once for every 100 records.
2819          if (countRecord % 100 == 0) {
2820            errors.progress();
2821          }
2822          countRecord++;
2823          return true;
2824        } catch (RuntimeException e) {
2825          LOG.error("Result=" + result);
2826          throw e;
2827        }
2828      }
2829    };
2830    if (!checkMetaOnly) {
2831      // Scan hbase:meta to pick up user regions
2832      MetaTableAccessor.fullScanRegions(connection, visitor);
2833    }
2834
2835    errors.print("");
2836    return true;
2837  }
2838
2839  /**
2840   * Prints summary of all tables found on the system.
2841   */
2842  private void printTableSummary(SortedMap<TableName, HbckTableInfo> tablesInfo) {
2843    StringBuilder sb = new StringBuilder();
2844    int numOfSkippedRegions;
2845    errors.print("Summary:");
2846    for (HbckTableInfo tInfo : tablesInfo.values()) {
2847      numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName())) ?
2848          skippedRegions.get(tInfo.getName()).size() : 0;
2849
2850      if (errors.tableHasErrors(tInfo)) {
2851        errors.print("Table " + tInfo.getName() + " is inconsistent.");
2852      } else if (numOfSkippedRegions > 0){
2853        errors.print("Table " + tInfo.getName() + " is okay (with "
2854          + numOfSkippedRegions + " skipped regions).");
2855      }
2856      else {
2857        errors.print("Table " + tInfo.getName() + " is okay.");
2858      }
2859      errors.print("    Number of regions: " + tInfo.getNumRegions());
2860      if (numOfSkippedRegions > 0) {
2861        Set<String> skippedRegionStrings = skippedRegions.get(tInfo.getName());
2862        System.out.println("    Number of skipped regions: " + numOfSkippedRegions);
2863        System.out.println("      List of skipped regions:");
2864        for(String sr : skippedRegionStrings) {
2865          System.out.println("        " + sr);
2866        }
2867      }
2868      sb.setLength(0); // clear out existing buffer, if any.
2869      sb.append("    Deployed on: ");
2870      for (ServerName server : tInfo.deployedOn) {
2871        sb.append(" " + server.toString());
2872      }
2873      errors.print(sb.toString());
2874    }
2875  }
2876
2877  static HbckErrorReporter getErrorReporter(final Configuration conf)
2878      throws ClassNotFoundException {
2879    Class<? extends HbckErrorReporter> reporter =
2880        conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class,
2881            HbckErrorReporter.class);
2882    return ReflectionUtils.newInstance(reporter, conf);
2883  }
2884
2885  static class PrintingErrorReporter implements HbckErrorReporter {
2886    public int errorCount = 0;
2887    private int showProgress;
2888    // How frequently calls to progress() will create output
2889    private static final int progressThreshold = 100;
2890
2891    Set<HbckTableInfo> errorTables = new HashSet<>();
2892
2893    // for use by unit tests to verify which errors were discovered
2894    private ArrayList<ERROR_CODE> errorList = new ArrayList<>();
2895
2896    @Override
2897    public void clear() {
2898      errorTables.clear();
2899      errorList.clear();
2900      errorCount = 0;
2901    }
2902
2903    @Override
2904    public synchronized void reportError(ERROR_CODE errorCode, String message) {
2905      if (errorCode == ERROR_CODE.WRONG_USAGE) {
2906        System.err.println(message);
2907        return;
2908      }
2909
2910      errorList.add(errorCode);
2911      if (!summary) {
2912        System.out.println("ERROR: " + message);
2913      }
2914      errorCount++;
2915      showProgress = 0;
2916    }
2917
2918    @Override
2919    public synchronized void reportError(ERROR_CODE errorCode, String message,
2920        HbckTableInfo table) {
2921      errorTables.add(table);
2922      reportError(errorCode, message);
2923    }
2924
2925    @Override
2926    public synchronized void reportError(ERROR_CODE errorCode, String message, HbckTableInfo table,
2927                                         HbckRegionInfo info) {
2928      errorTables.add(table);
2929      String reference = "(region " + info.getRegionNameAsString() + ")";
2930      reportError(errorCode, reference + " " + message);
2931    }
2932
2933    @Override
2934    public synchronized void reportError(ERROR_CODE errorCode, String message, HbckTableInfo table,
2935                                         HbckRegionInfo info1, HbckRegionInfo info2) {
2936      errorTables.add(table);
2937      String reference = "(regions " + info1.getRegionNameAsString()
2938          + " and " + info2.getRegionNameAsString() + ")";
2939      reportError(errorCode, reference + " " + message);
2940    }
2941
2942    @Override
2943    public synchronized void reportError(String message) {
2944      reportError(ERROR_CODE.UNKNOWN, message);
2945    }
2946
2947    /**
2948     * Report error information, but do not increment the error count.  Intended for cases
2949     * where the actual error would have been reported previously.
2950     * @param message
2951     */
2952    @Override
2953    public synchronized void report(String message) {
2954      if (! summary) {
2955        System.out.println("ERROR: " + message);
2956      }
2957      showProgress = 0;
2958    }
2959
2960    @Override
2961    public synchronized int summarize() {
2962      System.out.println(Integer.toString(errorCount) +
2963                         " inconsistencies detected.");
2964      if (errorCount == 0) {
2965        System.out.println("Status: OK");
2966        return 0;
2967      } else {
2968        System.out.println("Status: INCONSISTENT");
2969        return -1;
2970      }
2971    }
2972
2973    @Override
2974    public ArrayList<ERROR_CODE> getErrorList() {
2975      return errorList;
2976    }
2977
2978    @Override
2979    public synchronized void print(String message) {
2980      if (!summary) {
2981        System.out.println(message);
2982      }
2983    }
2984
2985    @Override
2986    public boolean tableHasErrors(HbckTableInfo table) {
2987      return errorTables.contains(table);
2988    }
2989
2990    @Override
2991    public void resetErrors() {
2992      errorCount = 0;
2993    }
2994
2995    @Override
2996    public synchronized void detail(String message) {
2997      if (details) {
2998        System.out.println(message);
2999      }
3000      showProgress = 0;
3001    }
3002
3003    @Override
3004    public synchronized void progress() {
3005      if (showProgress++ == progressThreshold) {
3006        if (!summary) {
3007          System.out.print(".");
3008        }
3009        showProgress = 0;
3010      }
3011    }
3012  }
3013
3014  /**
3015   * Contact a region server and get all information from it
3016   */
3017  static class WorkItemRegion implements Callable<Void> {
3018    private final HBaseFsck hbck;
3019    private final ServerName rsinfo;
3020    private final HbckErrorReporter errors;
3021    private final ClusterConnection connection;
3022
3023    WorkItemRegion(HBaseFsck hbck, ServerName info, HbckErrorReporter errors,
3024        ClusterConnection connection) {
3025      this.hbck = hbck;
3026      this.rsinfo = info;
3027      this.errors = errors;
3028      this.connection = connection;
3029    }
3030
3031    @Override
3032    public synchronized Void call() throws IOException {
3033      errors.progress();
3034      try {
3035        BlockingInterface server = connection.getAdmin(rsinfo);
3036
3037        // list all online regions from this region server
3038        List<RegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
3039        regions = filterRegions(regions);
3040
3041        if (details) {
3042          errors.detail("RegionServer: " + rsinfo.getServerName() +
3043                           " number of regions: " + regions.size());
3044          for (RegionInfo rinfo: regions) {
3045            errors.detail("  " + rinfo.getRegionNameAsString() +
3046                             " id: " + rinfo.getRegionId() +
3047                             " encoded_name: " + rinfo.getEncodedName() +
3048                             " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
3049                             " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
3050          }
3051        }
3052
3053        // check to see if the existence of this region matches the region in META
3054
3055        for (RegionInfo r : regions) {
3056          HbckRegionInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
3057          hbi.addServer(r, rsinfo);
3058        }
3059      } catch (IOException e) {          // unable to connect to the region server.
3060        errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
3061          " Unable to fetch region information. " + e);
3062        throw e;
3063      }
3064      return null;
3065    }
3066
3067    private List<RegionInfo> filterRegions(List<RegionInfo> regions) {
3068      List<RegionInfo> ret = Lists.newArrayList();
3069      for (RegionInfo hri : regions) {
3070        if (hri.isMetaRegion() || (!hbck.checkMetaOnly
3071            && hbck.isTableIncluded(hri.getTable()))) {
3072          ret.add(hri);
3073        }
3074      }
3075      return ret;
3076    }
3077  }
3078
3079  /**
3080   * Contact hdfs and get all information about specified table directory into
3081   * regioninfo list.
3082   */
3083  class WorkItemHdfsDir implements Callable<Void> {
3084    private FileStatus tableDir;
3085    private HbckErrorReporter errors;
3086    private FileSystem fs;
3087
3088    WorkItemHdfsDir(FileSystem fs, HbckErrorReporter errors, FileStatus status) {
3089      this.fs = fs;
3090      this.tableDir = status;
3091      this.errors = errors;
3092    }
3093
3094    @Override
3095    public synchronized Void call() throws InterruptedException, ExecutionException {
3096      final Vector<Exception> exceptions = new Vector<>();
3097
3098      try {
3099        final FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
3100        final List<Future<?>> futures = new ArrayList<>(regionDirs.length);
3101
3102        for (final FileStatus regionDir : regionDirs) {
3103          errors.progress();
3104          final String encodedName = regionDir.getPath().getName();
3105          // ignore directories that aren't hexadecimal
3106          if (!encodedName.toLowerCase(Locale.ROOT).matches("[0-9a-f]+")) {
3107            continue;
3108          }
3109
3110          if (!exceptions.isEmpty()) {
3111            break;
3112          }
3113
3114          futures.add(executor.submit(new Runnable() {
3115            @Override
3116            public void run() {
3117              try {
3118                LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
3119
3120                Path regioninfoFile = new Path(regionDir.getPath(), HRegionFileSystem.REGION_INFO_FILE);
3121                boolean regioninfoFileExists = fs.exists(regioninfoFile);
3122
3123                if (!regioninfoFileExists) {
3124                  // As tables become larger it is more and more likely that by the time you
3125                  // reach a given region that it will be gone due to region splits/merges.
3126                  if (!fs.exists(regionDir.getPath())) {
3127                    LOG.warn("By the time we tried to process this region dir it was already gone: "
3128                        + regionDir.getPath());
3129                    return;
3130                  }
3131                }
3132
3133                HbckRegionInfo hbi = HBaseFsck.this.getOrCreateInfo(encodedName);
3134                HbckRegionInfo.HdfsEntry he = new HbckRegionInfo.HdfsEntry();
3135                synchronized (hbi) {
3136                  if (hbi.getHdfsRegionDir() != null) {
3137                    errors.print("Directory " + encodedName + " duplicate??" +
3138                                 hbi.getHdfsRegionDir());
3139                  }
3140
3141                  he.regionDir = regionDir.getPath();
3142                  he.regionDirModTime = regionDir.getModificationTime();
3143                  he.hdfsRegioninfoFilePresent = regioninfoFileExists;
3144                  // we add to orphan list when we attempt to read .regioninfo
3145
3146                  // Set a flag if this region contains only edits
3147                  // This is special case if a region is left after split
3148                  he.hdfsOnlyEdits = true;
3149                  FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
3150                  Path ePath = WALSplitter.getRegionDirRecoveredEditsDir(regionDir.getPath());
3151                  for (FileStatus subDir : subDirs) {
3152                    errors.progress();
3153                    String sdName = subDir.getPath().getName();
3154                    if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
3155                      he.hdfsOnlyEdits = false;
3156                      break;
3157                    }
3158                  }
3159                  hbi.setHdfsEntry(he);
3160                }
3161              } catch (Exception e) {
3162                LOG.error("Could not load region dir", e);
3163                exceptions.add(e);
3164              }
3165            }
3166          }));
3167        }
3168
3169        // Ensure all pending tasks are complete (or that we run into an exception)
3170        for (Future<?> f : futures) {
3171          if (!exceptions.isEmpty()) {
3172            break;
3173          }
3174          try {
3175            f.get();
3176          } catch (ExecutionException e) {
3177            LOG.error("Unexpected exec exception!  Should've been caught already.  (Bug?)", e);
3178            // Shouldn't happen, we already logged/caught any exceptions in the Runnable
3179          };
3180        }
3181      } catch (IOException e) {
3182        LOG.error("Cannot execute WorkItemHdfsDir for " + tableDir, e);
3183        exceptions.add(e);
3184      } finally {
3185        if (!exceptions.isEmpty()) {
3186          errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
3187              + tableDir.getPath().getName()
3188              + " Unable to fetch all HDFS region information. ");
3189          // Just throw the first exception as an indication something bad happened
3190          // Don't need to propagate all the exceptions, we already logged them all anyway
3191          throw new ExecutionException("First exception in WorkItemHdfsDir", exceptions.firstElement());
3192        }
3193      }
3194      return null;
3195    }
3196  }
3197
3198  /**
3199   * Contact hdfs and get all information about specified table directory into
3200   * regioninfo list.
3201   */
3202  static class WorkItemHdfsRegionInfo implements Callable<Void> {
3203    private HbckRegionInfo hbi;
3204    private HBaseFsck hbck;
3205    private HbckErrorReporter errors;
3206
3207    WorkItemHdfsRegionInfo(HbckRegionInfo hbi, HBaseFsck hbck, HbckErrorReporter errors) {
3208      this.hbi = hbi;
3209      this.hbck = hbck;
3210      this.errors = errors;
3211    }
3212
3213    @Override
3214    public synchronized Void call() throws IOException {
3215      // only load entries that haven't been loaded yet.
3216      if (hbi.getHdfsHRI() == null) {
3217        try {
3218          errors.progress();
3219          hbi.loadHdfsRegioninfo(hbck.getConf());
3220        } catch (IOException ioe) {
3221          String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
3222              + hbi.getTableName() + " in hdfs dir "
3223              + hbi.getHdfsRegionDir()
3224              + "!  It may be an invalid format or version file.  Treating as "
3225              + "an orphaned regiondir.";
3226          errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
3227          try {
3228            hbck.debugLsr(hbi.getHdfsRegionDir());
3229          } catch (IOException ioe2) {
3230            LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
3231            throw ioe2;
3232          }
3233          hbck.orphanHdfsDirs.add(hbi);
3234          throw ioe;
3235        }
3236      }
3237      return null;
3238    }
3239  };
3240
3241  /**
3242   * Display the full report from fsck. This displays all live and dead region
3243   * servers, and all known regions.
3244   */
3245  public static void setDisplayFullReport() {
3246    details = true;
3247  }
3248
3249  public static boolean shouldDisplayFullReport() {
3250    return details;
3251  }
3252
3253  /**
3254   * Set exclusive mode.
3255   */
3256  public static void setForceExclusive() {
3257    forceExclusive = true;
3258  }
3259
3260  /**
3261   * Only one instance of hbck can modify HBase at a time.
3262   */
3263  public boolean isExclusive() {
3264    return fixAny || forceExclusive;
3265  }
3266
3267  /**
3268   * Set summary mode.
3269   * Print only summary of the tables and status (OK or INCONSISTENT)
3270   */
3271  static void setSummary() {
3272    summary = true;
3273  }
3274
3275  /**
3276   * Set hbase:meta check mode.
3277   * Print only info about hbase:meta table deployment/state
3278   */
3279  void setCheckMetaOnly() {
3280    checkMetaOnly = true;
3281  }
3282
3283  /**
3284   * Set region boundaries check mode.
3285   */
3286  void setRegionBoundariesCheck() {
3287    checkRegionBoundaries = true;
3288  }
3289
3290  /**
3291   * Set replication fix mode.
3292   */
3293  public void setFixReplication(boolean shouldFix) {
3294    fixReplication = shouldFix;
3295    fixAny |= shouldFix;
3296  }
3297
3298  public void setCleanReplicationBarrier(boolean shouldClean) {
3299    cleanReplicationBarrier = shouldClean;
3300  }
3301
3302  /**
3303   * Check if we should rerun fsck again. This checks if we've tried to
3304   * fix something and we should rerun fsck tool again.
3305   * Display the full report from fsck. This displays all live and dead
3306   * region servers, and all known regions.
3307   */
3308  void setShouldRerun() {
3309    rerun = true;
3310  }
3311
3312  public boolean shouldRerun() {
3313    return rerun;
3314  }
3315
3316  /**
3317   * Fix inconsistencies found by fsck. This should try to fix errors (if any)
3318   * found by fsck utility.
3319   */
3320  public void setFixAssignments(boolean shouldFix) {
3321    fixAssignments = shouldFix;
3322    fixAny |= shouldFix;
3323  }
3324
3325  boolean shouldFixAssignments() {
3326    return fixAssignments;
3327  }
3328
3329  public void setFixMeta(boolean shouldFix) {
3330    fixMeta = shouldFix;
3331    fixAny |= shouldFix;
3332  }
3333
3334  boolean shouldFixMeta() {
3335    return fixMeta;
3336  }
3337
3338  public void setFixEmptyMetaCells(boolean shouldFix) {
3339    fixEmptyMetaCells = shouldFix;
3340    fixAny |= shouldFix;
3341  }
3342
3343  boolean shouldFixEmptyMetaCells() {
3344    return fixEmptyMetaCells;
3345  }
3346
3347  public void setCheckHdfs(boolean checking) {
3348    checkHdfs = checking;
3349  }
3350
3351  boolean shouldCheckHdfs() {
3352    return checkHdfs;
3353  }
3354
3355  public void setFixHdfsHoles(boolean shouldFix) {
3356    fixHdfsHoles = shouldFix;
3357    fixAny |= shouldFix;
3358  }
3359
3360  boolean shouldFixHdfsHoles() {
3361    return fixHdfsHoles;
3362  }
3363
3364  public void setFixTableOrphans(boolean shouldFix) {
3365    fixTableOrphans = shouldFix;
3366    fixAny |= shouldFix;
3367  }
3368
3369  boolean shouldFixTableOrphans() {
3370    return fixTableOrphans;
3371  }
3372
3373  public void setFixHdfsOverlaps(boolean shouldFix) {
3374    fixHdfsOverlaps = shouldFix;
3375    fixAny |= shouldFix;
3376  }
3377
3378  boolean shouldFixHdfsOverlaps() {
3379    return fixHdfsOverlaps;
3380  }
3381
3382  public void setFixHdfsOrphans(boolean shouldFix) {
3383    fixHdfsOrphans = shouldFix;
3384    fixAny |= shouldFix;
3385  }
3386
3387  boolean shouldFixHdfsOrphans() {
3388    return fixHdfsOrphans;
3389  }
3390
3391  public void setFixVersionFile(boolean shouldFix) {
3392    fixVersionFile = shouldFix;
3393    fixAny |= shouldFix;
3394  }
3395
3396  public boolean shouldFixVersionFile() {
3397    return fixVersionFile;
3398  }
3399
3400  public void setSidelineBigOverlaps(boolean sbo) {
3401    this.sidelineBigOverlaps = sbo;
3402  }
3403
3404  public boolean shouldSidelineBigOverlaps() {
3405    return sidelineBigOverlaps;
3406  }
3407
3408  public void setFixSplitParents(boolean shouldFix) {
3409    fixSplitParents = shouldFix;
3410    fixAny |= shouldFix;
3411  }
3412
3413  public void setRemoveParents(boolean shouldFix) {
3414    removeParents = shouldFix;
3415    fixAny |= shouldFix;
3416  }
3417
3418  boolean shouldFixSplitParents() {
3419    return fixSplitParents;
3420  }
3421
3422  boolean shouldRemoveParents() {
3423    return removeParents;
3424  }
3425
3426  public void setFixReferenceFiles(boolean shouldFix) {
3427    fixReferenceFiles = shouldFix;
3428    fixAny |= shouldFix;
3429  }
3430
3431  boolean shouldFixReferenceFiles() {
3432    return fixReferenceFiles;
3433  }
3434
3435  public void setFixHFileLinks(boolean shouldFix) {
3436    fixHFileLinks = shouldFix;
3437    fixAny |= shouldFix;
3438  }
3439
3440  boolean shouldFixHFileLinks() {
3441    return fixHFileLinks;
3442  }
3443
3444  public boolean shouldIgnorePreCheckPermission() {
3445    return !fixAny || ignorePreCheckPermission;
3446  }
3447
3448  public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
3449    this.ignorePreCheckPermission = ignorePreCheckPermission;
3450  }
3451
3452  /**
3453   * @param mm maximum number of regions to merge into a single region.
3454   */
3455  public void setMaxMerge(int mm) {
3456    this.maxMerge = mm;
3457  }
3458
3459  public int getMaxMerge() {
3460    return maxMerge;
3461  }
3462
3463  public void setMaxOverlapsToSideline(int mo) {
3464    this.maxOverlapsToSideline = mo;
3465  }
3466
3467  public int getMaxOverlapsToSideline() {
3468    return maxOverlapsToSideline;
3469  }
3470
3471  /**
3472   * Only check/fix tables specified by the list,
3473   * Empty list means all tables are included.
3474   */
3475  boolean isTableIncluded(TableName table) {
3476    return (tablesIncluded.isEmpty()) || tablesIncluded.contains(table);
3477  }
3478
3479  public void includeTable(TableName table) {
3480    tablesIncluded.add(table);
3481  }
3482
3483  Set<TableName> getIncludedTables() {
3484    return new HashSet<>(tablesIncluded);
3485  }
3486
3487  /**
3488   * We are interested in only those tables that have not changed their state in
3489   * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
3490   * @param seconds - the time in seconds
3491   */
3492  public void setTimeLag(long seconds) {
3493    timelag = seconds * 1000; // convert to milliseconds
3494  }
3495
3496  /**
3497   *
3498   * @param sidelineDir - HDFS path to sideline data
3499   */
3500  public void setSidelineDir(String sidelineDir) {
3501    this.sidelineDir = new Path(sidelineDir);
3502  }
3503
3504  protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
3505    return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
3506  }
3507
3508  public HFileCorruptionChecker getHFilecorruptionChecker() {
3509    return hfcc;
3510  }
3511
3512  public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
3513    this.hfcc = hfcc;
3514  }
3515
3516  public void setRetCode(int code) {
3517    this.retcode = code;
3518  }
3519
3520  public int getRetCode() {
3521    return retcode;
3522  }
3523
3524  protected HBaseFsck printUsageAndExit() {
3525    StringWriter sw = new StringWriter(2048);
3526    PrintWriter out = new PrintWriter(sw);
3527    out.println("");
3528    out.println("-----------------------------------------------------------------------");
3529    out.println("NOTE: As of HBase version 2.0, the hbck tool is significantly changed.");
3530    out.println("In general, all Read-Only options are supported and can be be used");
3531    out.println("safely. Most -fix/ -repair options are NOT supported. Please see usage");
3532    out.println("below for details on which options are not supported.");
3533    out.println("-----------------------------------------------------------------------");
3534    out.println("");
3535    out.println("Usage: fsck [opts] {only tables}");
3536    out.println(" where [opts] are:");
3537    out.println("   -help Display help options (this)");
3538    out.println("   -details Display full report of all regions.");
3539    out.println("   -timelag <timeInSeconds>  Process only regions that " +
3540                       " have not experienced any metadata updates in the last " +
3541                       " <timeInSeconds> seconds.");
3542    out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
3543        " before checking if the fix worked if run with -fix");
3544    out.println("   -summary Print only summary of the tables and status.");
3545    out.println("   -metaonly Only check the state of the hbase:meta table.");
3546    out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
3547    out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
3548    out.println("   -exclusive Abort if another hbck is exclusive or fixing.");
3549
3550    out.println("");
3551    out.println("  Datafile Repair options: (expert features, use with caution!)");
3552    out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
3553    out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
3554
3555    out.println("");
3556    out.println(" Replication options");
3557    out.println("   -fixReplication   Deletes replication queues for removed peers");
3558
3559    out.println("");
3560    out.println("  Metadata Repair options supported as of version 2.0: (expert features, use with caution!)");
3561    out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
3562    out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
3563    out.println("   -fixHFileLinks  Try to offline lingering HFileLinks");
3564    out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
3565        + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
3566    out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
3567
3568    out.println("");
3569    out.println("NOTE: Following options are NOT supported as of HBase version 2.0+.");
3570    out.println("");
3571    out.println("  UNSUPPORTED Metadata Repair options: (expert features, use with caution!)");
3572    out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
3573    out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
3574    out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
3575    out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
3576    out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
3577    out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
3578    out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
3579    out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
3580    out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
3581    out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
3582    out.println("   -fixSplitParents  Try to force offline split parents to be online.");
3583    out.println("   -removeParents    Try to offline and sideline lingering parents and keep daughter regions.");
3584    out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
3585        + " (empty REGIONINFO_QUALIFIER rows)");
3586
3587    out.println("");
3588    out.println("  UNSUPPORTED Metadata Repair shortcuts");
3589    out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
3590        "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles" +
3591        "-fixHFileLinks");
3592    out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
3593    out.println("");
3594    out.println(" Replication options");
3595    out.println("   -fixReplication   Deletes replication queues for removed peers");
3596    out.println("   -cleanReplicationBrarier [tableName] clean the replication barriers " +
3597        "of a specified table, tableName is required");
3598    out.flush();
3599    errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
3600
3601    setRetCode(-2);
3602    return this;
3603  }
3604
3605  /**
3606   * Main program
3607   *
3608   * @param args
3609   * @throws Exception
3610   */
3611  public static void main(String[] args) throws Exception {
3612    // create a fsck object
3613    Configuration conf = HBaseConfiguration.create();
3614    Path hbasedir = FSUtils.getRootDir(conf);
3615    URI defaultFs = hbasedir.getFileSystem(conf).getUri();
3616    FSUtils.setFsDefault(conf, new Path(defaultFs));
3617    int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
3618    System.exit(ret);
3619  }
3620
3621  /**
3622   * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
3623   */
3624  static class HBaseFsckTool extends Configured implements Tool {
3625    HBaseFsckTool(Configuration conf) { super(conf); }
3626    @Override
3627    public int run(String[] args) throws Exception {
3628      HBaseFsck hbck = new HBaseFsck(getConf());
3629      hbck.exec(hbck.executor, args);
3630      hbck.close();
3631      return hbck.getRetCode();
3632    }
3633  };
3634
3635
3636  public HBaseFsck exec(ExecutorService exec, String[] args)
3637      throws KeeperException, IOException, InterruptedException, ReplicationException {
3638    long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
3639
3640    boolean checkCorruptHFiles = false;
3641    boolean sidelineCorruptHFiles = false;
3642
3643    // Process command-line args.
3644    for (int i = 0; i < args.length; i++) {
3645      String cmd = args[i];
3646      if (cmd.equals("-help") || cmd.equals("-h")) {
3647        return printUsageAndExit();
3648      } else if (cmd.equals("-details")) {
3649        setDisplayFullReport();
3650      } else if (cmd.equals("-exclusive")) {
3651        setForceExclusive();
3652      } else if (cmd.equals("-timelag")) {
3653        if (i == args.length - 1) {
3654          errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
3655          return printUsageAndExit();
3656        }
3657        try {
3658          long timelag = Long.parseLong(args[++i]);
3659          setTimeLag(timelag);
3660        } catch (NumberFormatException e) {
3661          errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
3662          return printUsageAndExit();
3663        }
3664      } else if (cmd.equals("-sleepBeforeRerun")) {
3665        if (i == args.length - 1) {
3666          errors.reportError(ERROR_CODE.WRONG_USAGE,
3667            "HBaseFsck: -sleepBeforeRerun needs a value.");
3668          return printUsageAndExit();
3669        }
3670        try {
3671          sleepBeforeRerun = Long.parseLong(args[++i]);
3672        } catch (NumberFormatException e) {
3673          errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
3674          return printUsageAndExit();
3675        }
3676      } else if (cmd.equals("-sidelineDir")) {
3677        if (i == args.length - 1) {
3678          errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
3679          return printUsageAndExit();
3680        }
3681        setSidelineDir(args[++i]);
3682      } else if (cmd.equals("-fix")) {
3683        errors.reportError(ERROR_CODE.WRONG_USAGE,
3684          "This option is deprecated, please use  -fixAssignments instead.");
3685        setFixAssignments(true);
3686      } else if (cmd.equals("-fixAssignments")) {
3687        setFixAssignments(true);
3688      } else if (cmd.equals("-fixMeta")) {
3689        setFixMeta(true);
3690      } else if (cmd.equals("-noHdfsChecking")) {
3691        setCheckHdfs(false);
3692      } else if (cmd.equals("-fixHdfsHoles")) {
3693        setFixHdfsHoles(true);
3694      } else if (cmd.equals("-fixHdfsOrphans")) {
3695        setFixHdfsOrphans(true);
3696      } else if (cmd.equals("-fixTableOrphans")) {
3697        setFixTableOrphans(true);
3698      } else if (cmd.equals("-fixHdfsOverlaps")) {
3699        setFixHdfsOverlaps(true);
3700      } else if (cmd.equals("-fixVersionFile")) {
3701        setFixVersionFile(true);
3702      } else if (cmd.equals("-sidelineBigOverlaps")) {
3703        setSidelineBigOverlaps(true);
3704      } else if (cmd.equals("-fixSplitParents")) {
3705        setFixSplitParents(true);
3706      } else if (cmd.equals("-removeParents")) {
3707        setRemoveParents(true);
3708      } else if (cmd.equals("-ignorePreCheckPermission")) {
3709        setIgnorePreCheckPermission(true);
3710      } else if (cmd.equals("-checkCorruptHFiles")) {
3711        checkCorruptHFiles = true;
3712      } else if (cmd.equals("-sidelineCorruptHFiles")) {
3713        sidelineCorruptHFiles = true;
3714      } else if (cmd.equals("-fixReferenceFiles")) {
3715        setFixReferenceFiles(true);
3716      } else if (cmd.equals("-fixHFileLinks")) {
3717        setFixHFileLinks(true);
3718      } else if (cmd.equals("-fixEmptyMetaCells")) {
3719        setFixEmptyMetaCells(true);
3720      } else if (cmd.equals("-repair")) {
3721        // this attempts to merge overlapping hdfs regions, needs testing
3722        // under load
3723        setFixHdfsHoles(true);
3724        setFixHdfsOrphans(true);
3725        setFixMeta(true);
3726        setFixAssignments(true);
3727        setFixHdfsOverlaps(true);
3728        setFixVersionFile(true);
3729        setSidelineBigOverlaps(true);
3730        setFixSplitParents(false);
3731        setCheckHdfs(true);
3732        setFixReferenceFiles(true);
3733        setFixHFileLinks(true);
3734      } else if (cmd.equals("-repairHoles")) {
3735        // this will make all missing hdfs regions available but may lose data
3736        setFixHdfsHoles(true);
3737        setFixHdfsOrphans(false);
3738        setFixMeta(true);
3739        setFixAssignments(true);
3740        setFixHdfsOverlaps(false);
3741        setSidelineBigOverlaps(false);
3742        setFixSplitParents(false);
3743        setCheckHdfs(true);
3744      } else if (cmd.equals("-maxOverlapsToSideline")) {
3745        if (i == args.length - 1) {
3746          errors.reportError(ERROR_CODE.WRONG_USAGE,
3747            "-maxOverlapsToSideline needs a numeric value argument.");
3748          return printUsageAndExit();
3749        }
3750        try {
3751          int maxOverlapsToSideline = Integer.parseInt(args[++i]);
3752          setMaxOverlapsToSideline(maxOverlapsToSideline);
3753        } catch (NumberFormatException e) {
3754          errors.reportError(ERROR_CODE.WRONG_USAGE,
3755            "-maxOverlapsToSideline needs a numeric value argument.");
3756          return printUsageAndExit();
3757        }
3758      } else if (cmd.equals("-maxMerge")) {
3759        if (i == args.length - 1) {
3760          errors.reportError(ERROR_CODE.WRONG_USAGE,
3761            "-maxMerge needs a numeric value argument.");
3762          return printUsageAndExit();
3763        }
3764        try {
3765          int maxMerge = Integer.parseInt(args[++i]);
3766          setMaxMerge(maxMerge);
3767        } catch (NumberFormatException e) {
3768          errors.reportError(ERROR_CODE.WRONG_USAGE,
3769            "-maxMerge needs a numeric value argument.");
3770          return printUsageAndExit();
3771        }
3772      } else if (cmd.equals("-summary")) {
3773        setSummary();
3774      } else if (cmd.equals("-metaonly")) {
3775        setCheckMetaOnly();
3776      } else if (cmd.equals("-boundaries")) {
3777        setRegionBoundariesCheck();
3778      } else if (cmd.equals("-fixReplication")) {
3779        setFixReplication(true);
3780      } else if (cmd.equals("-cleanReplicationBarrier")) {
3781        setCleanReplicationBarrier(true);
3782        if(args[++i].startsWith("-")){
3783          printUsageAndExit();
3784        }
3785        setCleanReplicationBarrierTable(args[i]);
3786      } else if (cmd.startsWith("-")) {
3787        errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
3788        return printUsageAndExit();
3789      } else {
3790        includeTable(TableName.valueOf(cmd));
3791        errors.print("Allow checking/fixes for table: " + cmd);
3792      }
3793    }
3794
3795    errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
3796
3797    // pre-check current user has FS write permission or not
3798    try {
3799      preCheckPermission();
3800    } catch (AccessDeniedException ace) {
3801      Runtime.getRuntime().exit(-1);
3802    } catch (IOException ioe) {
3803      Runtime.getRuntime().exit(-1);
3804    }
3805
3806    // do the real work of hbck
3807    connect();
3808
3809    // after connecting to server above, we have server version
3810    // check if unsupported option is specified based on server version
3811    if (!isOptionsSupported(args)) {
3812      return printUsageAndExit();
3813    }
3814
3815    try {
3816      // if corrupt file mode is on, first fix them since they may be opened later
3817      if (checkCorruptHFiles || sidelineCorruptHFiles) {
3818        LOG.info("Checking all hfiles for corruption");
3819        HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
3820        setHFileCorruptionChecker(hfcc); // so we can get result
3821        Collection<TableName> tables = getIncludedTables();
3822        Collection<Path> tableDirs = new ArrayList<>();
3823        Path rootdir = FSUtils.getRootDir(getConf());
3824        if (tables.size() > 0) {
3825          for (TableName t : tables) {
3826            tableDirs.add(FSUtils.getTableDir(rootdir, t));
3827          }
3828        } else {
3829          tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
3830        }
3831        hfcc.checkTables(tableDirs);
3832        hfcc.report(errors);
3833      }
3834
3835      // check and fix table integrity, region consistency.
3836      int code = onlineHbck();
3837      setRetCode(code);
3838      // If we have changed the HBase state it is better to run hbck again
3839      // to see if we haven't broken something else in the process.
3840      // We run it only once more because otherwise we can easily fall into
3841      // an infinite loop.
3842      if (shouldRerun()) {
3843        try {
3844          LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
3845          Thread.sleep(sleepBeforeRerun);
3846        } catch (InterruptedException ie) {
3847          LOG.warn("Interrupted while sleeping");
3848          return this;
3849        }
3850        // Just report
3851        setFixAssignments(false);
3852        setFixMeta(false);
3853        setFixHdfsHoles(false);
3854        setFixHdfsOverlaps(false);
3855        setFixVersionFile(false);
3856        setFixTableOrphans(false);
3857        errors.resetErrors();
3858        code = onlineHbck();
3859        setRetCode(code);
3860      }
3861    } finally {
3862      IOUtils.closeQuietly(this);
3863    }
3864    return this;
3865  }
3866
3867  private boolean isOptionsSupported(String[] args) {
3868    boolean result = true;
3869    String hbaseServerVersion = status.getHBaseVersion();
3870    if (VersionInfo.compareVersion("2.any.any", hbaseServerVersion) < 0) {
3871      // Process command-line args.
3872      for (String arg : args) {
3873        if (unsupportedOptionsInV2.contains(arg)) {
3874          errors.reportError(ERROR_CODE.UNSUPPORTED_OPTION,
3875              "option '" + arg + "' is not " + "supportted!");
3876          result = false;
3877          break;
3878        }
3879      }
3880    }
3881    return result;
3882  }
3883
3884  public void setCleanReplicationBarrierTable(String cleanReplicationBarrierTable) {
3885    this.cleanReplicationBarrierTable = TableName.valueOf(cleanReplicationBarrierTable);
3886  }
3887
3888  public void cleanReplicationBarrier() throws IOException {
3889    if (!cleanReplicationBarrier || cleanReplicationBarrierTable == null) {
3890      return;
3891    }
3892    if (cleanReplicationBarrierTable.isSystemTable()) {
3893      errors.reportError(ERROR_CODE.INVALID_TABLE,
3894        "invalid table: " + cleanReplicationBarrierTable);
3895      return;
3896    }
3897
3898    boolean isGlobalScope = false;
3899    try {
3900      isGlobalScope = admin.getDescriptor(cleanReplicationBarrierTable).hasGlobalReplicationScope();
3901    } catch (TableNotFoundException e) {
3902      LOG.info("we may need to clean some erroneous data due to bugs");
3903    }
3904
3905    if (isGlobalScope) {
3906      errors.reportError(ERROR_CODE.INVALID_TABLE,
3907        "table's replication scope is global: " + cleanReplicationBarrierTable);
3908      return;
3909    }
3910    List<byte[]> regionNames = new ArrayList<>();
3911    Scan barrierScan = new Scan();
3912    barrierScan.setCaching(100);
3913    barrierScan.addFamily(HConstants.REPLICATION_BARRIER_FAMILY);
3914    barrierScan
3915        .withStartRow(MetaTableAccessor.getTableStartRowForMeta(cleanReplicationBarrierTable,
3916          MetaTableAccessor.QueryType.REGION))
3917        .withStopRow(MetaTableAccessor.getTableStopRowForMeta(cleanReplicationBarrierTable,
3918          MetaTableAccessor.QueryType.REGION));
3919    Result result;
3920    try (ResultScanner scanner = meta.getScanner(barrierScan)) {
3921      while ((result = scanner.next()) != null) {
3922        regionNames.add(result.getRow());
3923      }
3924    }
3925    if (regionNames.size() <= 0) {
3926      errors.reportError(ERROR_CODE.INVALID_TABLE,
3927        "there is no barriers of this table: " + cleanReplicationBarrierTable);
3928      return;
3929    }
3930    ReplicationQueueStorage queueStorage =
3931        ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
3932    List<ReplicationPeerDescription> peerDescriptions = admin.listReplicationPeers();
3933    if (peerDescriptions != null && peerDescriptions.size() > 0) {
3934      List<String> peers = peerDescriptions.stream()
3935          .filter(peerConfig -> peerConfig.getPeerConfig()
3936            .needToReplicate(cleanReplicationBarrierTable))
3937          .map(peerConfig -> peerConfig.getPeerId()).collect(Collectors.toList());
3938      try {
3939        List<String> batch = new ArrayList<>();
3940        for (String peer : peers) {
3941          for (byte[] regionName : regionNames) {
3942            batch.add(RegionInfo.encodeRegionName(regionName));
3943            if (batch.size() % 100 == 0) {
3944              queueStorage.removeLastSequenceIds(peer, batch);
3945              batch.clear();
3946            }
3947          }
3948          if (batch.size() > 0) {
3949            queueStorage.removeLastSequenceIds(peer, batch);
3950            batch.clear();
3951          }
3952        }
3953      } catch (ReplicationException re) {
3954        throw new IOException(re);
3955      }
3956    }
3957    for (byte[] regionName : regionNames) {
3958      meta.delete(new Delete(regionName).addFamily(HConstants.REPLICATION_BARRIER_FAMILY));
3959    }
3960    setShouldRerun();
3961  }
3962
3963  /**
3964   * ls -r for debugging purposes
3965   */
3966  void debugLsr(Path p) throws IOException {
3967    debugLsr(getConf(), p, errors);
3968  }
3969
3970  /**
3971   * ls -r for debugging purposes
3972   */
3973  public static void debugLsr(Configuration conf,
3974      Path p) throws IOException {
3975    debugLsr(conf, p, new PrintingErrorReporter());
3976  }
3977
3978  /**
3979   * ls -r for debugging purposes
3980   */
3981  public static void debugLsr(Configuration conf,
3982      Path p, HbckErrorReporter errors) throws IOException {
3983    if (!LOG.isDebugEnabled() || p == null) {
3984      return;
3985    }
3986    FileSystem fs = p.getFileSystem(conf);
3987
3988    if (!fs.exists(p)) {
3989      // nothing
3990      return;
3991    }
3992    errors.print(p.toString());
3993
3994    if (fs.isFile(p)) {
3995      return;
3996    }
3997
3998    if (fs.getFileStatus(p).isDirectory()) {
3999      FileStatus[] fss= fs.listStatus(p);
4000      for (FileStatus status : fss) {
4001        debugLsr(conf, status.getPath(), errors);
4002      }
4003    }
4004  }
4005}