001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import java.io.Closeable;
021import java.io.FileNotFoundException;
022import java.io.IOException;
023import java.io.InterruptedIOException;
024import java.io.PrintWriter;
025import java.io.StringWriter;
026import java.net.InetAddress;
027import java.net.URI;
028import java.util.ArrayList;
029import java.util.Collection;
030import java.util.Collections;
031import java.util.Comparator;
032import java.util.EnumSet;
033import java.util.HashMap;
034import java.util.HashSet;
035import java.util.Iterator;
036import java.util.List;
037import java.util.Locale;
038import java.util.Map;
039import java.util.Map.Entry;
040import java.util.Objects;
041import java.util.Optional;
042import java.util.Set;
043import java.util.SortedMap;
044import java.util.TreeMap;
045import java.util.Vector;
046import java.util.concurrent.Callable;
047import java.util.concurrent.ConcurrentSkipListMap;
048import java.util.concurrent.ExecutionException;
049import java.util.concurrent.ExecutorService;
050import java.util.concurrent.Executors;
051import java.util.concurrent.Future;
052import java.util.concurrent.FutureTask;
053import java.util.concurrent.ScheduledThreadPoolExecutor;
054import java.util.concurrent.TimeUnit;
055import java.util.concurrent.TimeoutException;
056import java.util.concurrent.atomic.AtomicBoolean;
057import java.util.concurrent.atomic.AtomicInteger;
058import java.util.stream.Collectors;
059
060import org.apache.commons.io.IOUtils;
061import org.apache.commons.lang3.StringUtils;
062import org.apache.hadoop.conf.Configuration;
063import org.apache.hadoop.conf.Configured;
064import org.apache.hadoop.fs.FSDataOutputStream;
065import org.apache.hadoop.fs.FileStatus;
066import org.apache.hadoop.fs.FileSystem;
067import org.apache.hadoop.fs.Path;
068import org.apache.hadoop.fs.permission.FsAction;
069import org.apache.hadoop.fs.permission.FsPermission;
070import org.apache.hadoop.hbase.Abortable;
071import org.apache.hadoop.hbase.Cell;
072import org.apache.hadoop.hbase.CellUtil;
073import org.apache.hadoop.hbase.ClusterMetrics;
074import org.apache.hadoop.hbase.ClusterMetrics.Option;
075import org.apache.hadoop.hbase.HBaseConfiguration;
076import org.apache.hadoop.hbase.HBaseInterfaceAudience;
077import org.apache.hadoop.hbase.HConstants;
078import org.apache.hadoop.hbase.HRegionLocation;
079import org.apache.hadoop.hbase.KeyValue;
080import org.apache.hadoop.hbase.MasterNotRunningException;
081import org.apache.hadoop.hbase.MetaTableAccessor;
082import org.apache.hadoop.hbase.RegionLocations;
083import org.apache.hadoop.hbase.ServerName;
084import org.apache.hadoop.hbase.TableName;
085import org.apache.hadoop.hbase.TableNotFoundException;
086import org.apache.hadoop.hbase.ZooKeeperConnectionException;
087import org.apache.hadoop.hbase.client.Admin;
088import org.apache.hadoop.hbase.client.ClusterConnection;
089import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
090import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
091import org.apache.hadoop.hbase.client.Connection;
092import org.apache.hadoop.hbase.client.ConnectionFactory;
093import org.apache.hadoop.hbase.client.Delete;
094import org.apache.hadoop.hbase.client.Get;
095import org.apache.hadoop.hbase.client.Put;
096import org.apache.hadoop.hbase.client.RegionInfo;
097import org.apache.hadoop.hbase.client.RegionInfoBuilder;
098import org.apache.hadoop.hbase.client.RegionReplicaUtil;
099import org.apache.hadoop.hbase.client.Result;
100import org.apache.hadoop.hbase.client.ResultScanner;
101import org.apache.hadoop.hbase.client.RowMutations;
102import org.apache.hadoop.hbase.client.Scan;
103import org.apache.hadoop.hbase.client.Table;
104import org.apache.hadoop.hbase.client.TableDescriptor;
105import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
106import org.apache.hadoop.hbase.client.TableState;
107import org.apache.hadoop.hbase.io.FileLink;
108import org.apache.hadoop.hbase.io.HFileLink;
109import org.apache.hadoop.hbase.io.hfile.CacheConfig;
110import org.apache.hadoop.hbase.io.hfile.HFile;
111import org.apache.hadoop.hbase.master.RegionState;
112import org.apache.hadoop.hbase.regionserver.HRegion;
113import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
114import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
115import org.apache.hadoop.hbase.replication.ReplicationException;
116import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
117import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
118import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
119import org.apache.hadoop.hbase.security.AccessDeniedException;
120import org.apache.hadoop.hbase.security.UserProvider;
121import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
122import org.apache.hadoop.hbase.util.HbckErrorReporter.ERROR_CODE;
123import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
124import org.apache.hadoop.hbase.util.hbck.ReplicationChecker;
125import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
126import org.apache.hadoop.hbase.wal.WALSplitUtil;
127import org.apache.hadoop.hbase.zookeeper.ZKUtil;
128import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
129import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
130import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
131import org.apache.hadoop.ipc.RemoteException;
132import org.apache.hadoop.security.UserGroupInformation;
133import org.apache.hadoop.util.ReflectionUtils;
134import org.apache.hadoop.util.Tool;
135import org.apache.hadoop.util.ToolRunner;
136import org.apache.yetus.audience.InterfaceAudience;
137import org.apache.yetus.audience.InterfaceStability;
138import org.apache.zookeeper.KeeperException;
139import org.slf4j.Logger;
140import org.slf4j.LoggerFactory;
141
142import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
143import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
144import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
145import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
146import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
147
148import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
149import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
150
151/**
152 * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
153 * table integrity problems in a corrupted HBase. This tool was written for hbase-1.x. It does not
154 * work with hbase-2.x; it can read state but is not allowed to change state; i.e. effect 'repair'.
155 * Even though it can 'read' state, given how so much has changed in how hbase1 and hbase2 operate,
156 * it will often misread. See hbck2 (HBASE-19121) for a hbck tool for hbase2. This class is
157 * deprecated.
158 *
159 * <p>
160 * Region consistency checks verify that hbase:meta, region deployment on region
161 * servers and the state of data in HDFS (.regioninfo files) all are in
162 * accordance.
163 * <p>
164 * Table integrity checks verify that all possible row keys resolve to exactly
165 * one region of a table.  This means there are no individual degenerate
166 * or backwards regions; no holes between regions; and that there are no
167 * overlapping regions.
168 * <p>
169 * The general repair strategy works in two phases:
170 * <ol>
171 * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
172 * <li> Repair Region Consistency with hbase:meta and assignments
173 * </ol>
174 * <p>
175 * For table integrity repairs, the tables' region directories are scanned
176 * for .regioninfo files.  Each table's integrity is then verified.  If there
177 * are any orphan regions (regions with no .regioninfo files) or holes, new
178 * regions are fabricated.  Backwards regions are sidelined as well as empty
179 * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
180 * a new region is created and all data is merged into the new region.
181 * <p>
182 * Table integrity repairs deal solely with HDFS and could potentially be done
183 * offline -- the hbase region servers or master do not need to be running.
184 * This phase can eventually be used to completely reconstruct the hbase:meta table in
185 * an offline fashion.
186 * <p>
187 * Region consistency requires three conditions -- 1) valid .regioninfo file
188 * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
189 * and 3) a region is deployed only at the regionserver that was assigned to
190 * with proper state in the master.
191 * <p>
192 * Region consistency repairs require hbase to be online so that hbck can
193 * contact the HBase master and region servers.  The hbck#connect() method must
194 * first be called successfully.  Much of the region consistency information
195 * is transient and less risky to repair.
196 * <p>
197 * If hbck is run from the command line, there are a handful of arguments that
198 * can be used to limit the kinds of repairs hbck will do.  See the code in
199 * {@link #printUsageAndExit()} for more details.
200 * @deprecated For removal in hbase-4.0.0. Use HBCK2 instead.
201 */
202@Deprecated
203@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
204@InterfaceStability.Evolving
205public class HBaseFsck extends Configured implements Closeable {
206  public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
207  public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
208  private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
209  private static boolean rsSupportsOffline = true;
210  private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
211  private static final int DEFAULT_MAX_MERGE = 5;
212
213  /**
214   * Here is where hbase-1.x used to default the lock for hbck1.
215   * It puts in place a lock when it goes to write/make changes.
216   */
217  @VisibleForTesting
218  public static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
219  private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5;
220  private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds
221  private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds
222  // We have to set the timeout value > HdfsConstants.LEASE_SOFTLIMIT_PERIOD.
223  // In HADOOP-2.6 and later, the Namenode proxy now created with custom RetryPolicy for
224  // AlreadyBeingCreatedException which is implies timeout on this operations up to
225  // HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds).
226  private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds
227  private static final int DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS = 5;
228  private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds
229  private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds
230
231  /**********************
232   * Internal resources
233   **********************/
234  private static final Logger LOG = LoggerFactory.getLogger(HBaseFsck.class.getName());
235  private ClusterMetrics status;
236  private ClusterConnection connection;
237  private Admin admin;
238  private Table meta;
239  // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
240  protected ExecutorService executor;
241  private long startMillis = EnvironmentEdgeManager.currentTime();
242  private HFileCorruptionChecker hfcc;
243  private int retcode = 0;
244  private Path HBCK_LOCK_PATH;
245  private FSDataOutputStream hbckOutFd;
246  // This lock is to prevent cleanup of balancer resources twice between
247  // ShutdownHook and the main code. We cleanup only if the connect() is
248  // successful
249  private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
250
251  // Unsupported options in HBase 2.0+
252  private static final Set<String> unsupportedOptionsInV2 = Sets.newHashSet("-fix",
253      "-fixAssignments", "-fixMeta", "-fixHdfsHoles", "-fixHdfsOrphans", "-fixTableOrphans",
254      "-fixHdfsOverlaps", "-sidelineBigOverlaps", "-fixSplitParents", "-removeParents",
255      "-fixEmptyMetaCells", "-repair", "-repairHoles", "-maxOverlapsToSideline", "-maxMerge");
256
257  /***********
258   * Options
259   ***********/
260  private static boolean details = false; // do we display the full report
261  private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
262  private static boolean forceExclusive = false; // only this hbck can modify HBase
263  private boolean fixAssignments = false; // fix assignment errors?
264  private boolean fixMeta = false; // fix meta errors?
265  private boolean checkHdfs = true; // load and check fs consistency?
266  private boolean fixHdfsHoles = false; // fix fs holes?
267  private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
268  private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
269  private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
270  private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
271  private boolean fixSplitParents = false; // fix lingering split parents
272  private boolean removeParents = false; // remove split parents
273  private boolean fixReferenceFiles = false; // fix lingering reference store file
274  private boolean fixHFileLinks = false; // fix lingering HFileLinks
275  private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
276  private boolean fixReplication = false; // fix undeleted replication queues for removed peer
277  private boolean cleanReplicationBarrier = false; // clean replication barriers of a table
278  private boolean fixAny = false; // Set to true if any of the fix is required.
279
280  // limit checking/fixes to listed tables, if empty attempt to check/fix all
281  // hbase:meta are always checked
282  private Set<TableName> tablesIncluded = new HashSet<>();
283  private TableName cleanReplicationBarrierTable;
284  private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
285  // maximum number of overlapping regions to sideline
286  private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE;
287  private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
288  private Path sidelineDir = null;
289
290  private boolean rerun = false; // if we tried to fix something, rerun hbck
291  private static boolean summary = false; // if we want to print less output
292  private boolean checkMetaOnly = false;
293  private boolean checkRegionBoundaries = false;
294  private boolean ignorePreCheckPermission = false; // if pre-check permission
295
296  /*********
297   * State
298   *********/
299  final private HbckErrorReporter errors;
300  int fixes = 0;
301
302  /**
303   * This map contains the state of all hbck items.  It maps from encoded region
304   * name to HbckRegionInfo structure.  The information contained in HbckRegionInfo is used
305   * to detect and correct consistency (hdfs/meta/deployment) problems.
306   */
307  private TreeMap<String, HbckRegionInfo> regionInfoMap = new TreeMap<>();
308  // Empty regioninfo qualifiers in hbase:meta
309  private Set<Result> emptyRegionInfoQualifiers = new HashSet<>();
310
311  /**
312   * This map from Tablename -> TableInfo contains the structures necessary to
313   * detect table consistency problems (holes, dupes, overlaps).  It is sorted
314   * to prevent dupes.
315   *
316   * If tablesIncluded is empty, this map contains all tables.
317   * Otherwise, it contains only meta tables and tables in tablesIncluded,
318   * unless checkMetaOnly is specified, in which case, it contains only
319   * the meta table
320   */
321  private SortedMap<TableName, HbckTableInfo> tablesInfo = new ConcurrentSkipListMap<>();
322
323  /**
324   * When initially looking at HDFS, we attempt to find any orphaned data.
325   */
326  private List<HbckRegionInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<>());
327
328  private Map<TableName, Set<String>> orphanTableDirs = new HashMap<>();
329  private Map<TableName, TableState> tableStates = new HashMap<>();
330  private final RetryCounterFactory lockFileRetryCounterFactory;
331  private final RetryCounterFactory createZNodeRetryCounterFactory;
332
333  private Map<TableName, Set<String>> skippedRegions = new HashMap<>();
334
335  private ZKWatcher zkw = null;
336  private String hbckEphemeralNodePath = null;
337  private boolean hbckZodeCreated = false;
338
339  /**
340   * Constructor
341   *
342   * @param conf Configuration object
343   * @throws MasterNotRunningException if the master is not running
344   * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
345   */
346  public HBaseFsck(Configuration conf) throws IOException, ClassNotFoundException {
347    this(conf, createThreadPool(conf));
348  }
349
350  private static ExecutorService createThreadPool(Configuration conf) {
351    int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
352    return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
353  }
354
355  /**
356   * Constructor
357   *
358   * @param conf
359   *          Configuration object
360   * @throws MasterNotRunningException
361   *           if the master is not running
362   * @throws ZooKeeperConnectionException
363   *           if unable to connect to ZooKeeper
364   */
365  public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
366      ZooKeeperConnectionException, IOException, ClassNotFoundException {
367    super(conf);
368    errors = getErrorReporter(getConf());
369    this.executor = exec;
370    lockFileRetryCounterFactory = createLockRetryCounterFactory(getConf());
371    createZNodeRetryCounterFactory = createZnodeRetryCounterFactory(getConf());
372    zkw = createZooKeeperWatcher();
373  }
374
375  /**
376   * @return A retry counter factory configured for retrying lock file creation.
377   */
378  public static RetryCounterFactory createLockRetryCounterFactory(Configuration conf) {
379    return new RetryCounterFactory(
380        conf.getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
381        conf.getInt("hbase.hbck.lockfile.attempt.sleep.interval",
382            DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),
383        conf.getInt("hbase.hbck.lockfile.attempt.maxsleeptime",
384            DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));
385  }
386
387  /**
388   * @return A retry counter factory configured for retrying znode creation.
389   */
390  private static RetryCounterFactory createZnodeRetryCounterFactory(Configuration conf) {
391    return new RetryCounterFactory(
392        conf.getInt("hbase.hbck.createznode.attempts", DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS),
393        conf.getInt("hbase.hbck.createznode.attempt.sleep.interval",
394            DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL),
395        conf.getInt("hbase.hbck.createznode.attempt.maxsleeptime",
396            DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME));
397  }
398
399  /**
400   * @return Return the tmp dir this tool writes too.
401   */
402  @VisibleForTesting
403  public static Path getTmpDir(Configuration conf) throws IOException {
404    return new Path(FSUtils.getRootDir(conf), HConstants.HBASE_TEMP_DIRECTORY);
405  }
406
407  private static class FileLockCallable implements Callable<FSDataOutputStream> {
408    RetryCounter retryCounter;
409    private final Configuration conf;
410    private Path hbckLockPath = null;
411
412    public FileLockCallable(Configuration conf, RetryCounter retryCounter) {
413      this.retryCounter = retryCounter;
414      this.conf = conf;
415    }
416
417    /**
418     * @return Will be <code>null</code> unless you call {@link #call()}
419     */
420    Path getHbckLockPath() {
421      return this.hbckLockPath;
422    }
423
424    @Override
425    public FSDataOutputStream call() throws IOException {
426      try {
427        FileSystem fs = FSUtils.getCurrentFileSystem(this.conf);
428        FsPermission defaultPerms = FSUtils.getFilePermissions(fs, this.conf,
429            HConstants.DATA_FILE_UMASK_KEY);
430        Path tmpDir = getTmpDir(conf);
431        this.hbckLockPath = new Path(tmpDir, HBCK_LOCK_FILE);
432        fs.mkdirs(tmpDir);
433        final FSDataOutputStream out = createFileWithRetries(fs, this.hbckLockPath, defaultPerms);
434        out.writeBytes(InetAddress.getLocalHost().toString());
435        // Add a note into the file we write on why hbase2 is writing out an hbck1 lock file.
436        out.writeBytes(" Written by an hbase-2.x Master to block an " +
437            "attempt by an hbase-1.x HBCK tool making modification to state. " +
438            "See 'HBCK must match HBase server version' in the hbase refguide.");
439        out.flush();
440        return out;
441      } catch(RemoteException e) {
442        if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
443          return null;
444        } else {
445          throw e;
446        }
447      }
448    }
449
450    private FSDataOutputStream createFileWithRetries(final FileSystem fs,
451        final Path hbckLockFilePath, final FsPermission defaultPerms)
452        throws IOException {
453      IOException exception = null;
454      do {
455        try {
456          return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false);
457        } catch (IOException ioe) {
458          LOG.info("Failed to create lock file " + hbckLockFilePath.getName()
459              + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of "
460              + retryCounter.getMaxAttempts());
461          LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(),
462              ioe);
463          try {
464            exception = ioe;
465            retryCounter.sleepUntilNextRetry();
466          } catch (InterruptedException ie) {
467            throw (InterruptedIOException) new InterruptedIOException(
468                "Can't create lock file " + hbckLockFilePath.getName())
469            .initCause(ie);
470          }
471        }
472      } while (retryCounter.shouldRetry());
473
474      throw exception;
475    }
476  }
477
478  /**
479   * This method maintains a lock using a file. If the creation fails we return null
480   *
481   * @return FSDataOutputStream object corresponding to the newly opened lock file
482   * @throws IOException if IO failure occurs
483   */
484  public static Pair<Path, FSDataOutputStream> checkAndMarkRunningHbck(Configuration conf,
485      RetryCounter retryCounter) throws IOException {
486    FileLockCallable callable = new FileLockCallable(conf, retryCounter);
487    ExecutorService executor = Executors.newFixedThreadPool(1);
488    FutureTask<FSDataOutputStream> futureTask = new FutureTask<>(callable);
489    executor.execute(futureTask);
490    final int timeoutInSeconds = conf.getInt(
491      "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT);
492    FSDataOutputStream stream = null;
493    try {
494      stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);
495    } catch (ExecutionException ee) {
496      LOG.warn("Encountered exception when opening lock file", ee);
497    } catch (InterruptedException ie) {
498      LOG.warn("Interrupted when opening lock file", ie);
499      Thread.currentThread().interrupt();
500    } catch (TimeoutException exception) {
501      // took too long to obtain lock
502      LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");
503      futureTask.cancel(true);
504    } finally {
505      executor.shutdownNow();
506    }
507    return new Pair<Path, FSDataOutputStream>(callable.getHbckLockPath(), stream);
508  }
509
510  private void unlockHbck() {
511    if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) {
512      RetryCounter retryCounter = lockFileRetryCounterFactory.create();
513      do {
514        try {
515          IOUtils.closeQuietly(hbckOutFd);
516          FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()), HBCK_LOCK_PATH, true);
517          LOG.info("Finishing hbck");
518          return;
519        } catch (IOException ioe) {
520          LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try="
521              + (retryCounter.getAttemptTimes() + 1) + " of "
522              + retryCounter.getMaxAttempts());
523          LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe);
524          try {
525            retryCounter.sleepUntilNextRetry();
526          } catch (InterruptedException ie) {
527            Thread.currentThread().interrupt();
528            LOG.warn("Interrupted while deleting lock file" +
529                HBCK_LOCK_PATH);
530            return;
531          }
532        }
533      } while (retryCounter.shouldRetry());
534    }
535  }
536
537  /**
538   * To repair region consistency, one must call connect() in order to repair
539   * online state.
540   */
541  public void connect() throws IOException {
542
543    if (isExclusive()) {
544      // Grab the lock
545      Pair<Path, FSDataOutputStream> pair =
546          checkAndMarkRunningHbck(getConf(), this.lockFileRetryCounterFactory.create());
547      HBCK_LOCK_PATH = pair.getFirst();
548      this.hbckOutFd = pair.getSecond();
549      if (hbckOutFd == null) {
550        setRetCode(-1);
551        LOG.error("Another instance of hbck is fixing HBase, exiting this instance. " +
552            "[If you are sure no other instance is running, delete the lock file " +
553            HBCK_LOCK_PATH + " and rerun the tool]");
554        throw new IOException("Duplicate hbck - Abort");
555      }
556
557      // Make sure to cleanup the lock
558      hbckLockCleanup.set(true);
559    }
560
561
562    // Add a shutdown hook to this thread, in case user tries to
563    // kill the hbck with a ctrl-c, we want to cleanup the lock so that
564    // it is available for further calls
565    Runtime.getRuntime().addShutdownHook(new Thread() {
566      @Override
567      public void run() {
568        IOUtils.closeQuietly(HBaseFsck.this);
569        cleanupHbckZnode();
570        unlockHbck();
571      }
572    });
573
574    LOG.info("Launching hbck");
575
576    connection = (ClusterConnection)ConnectionFactory.createConnection(getConf());
577    admin = connection.getAdmin();
578    meta = connection.getTable(TableName.META_TABLE_NAME);
579    status = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS,
580      Option.DEAD_SERVERS, Option.MASTER, Option.BACKUP_MASTERS,
581      Option.REGIONS_IN_TRANSITION, Option.HBASE_VERSION));
582  }
583
584  /**
585   * Get deployed regions according to the region servers.
586   */
587  private void loadDeployedRegions() throws IOException, InterruptedException {
588    // From the master, get a list of all known live region servers
589    Collection<ServerName> regionServers = status.getLiveServerMetrics().keySet();
590    errors.print("Number of live region servers: " + regionServers.size());
591    if (details) {
592      for (ServerName rsinfo: regionServers) {
593        errors.print("  " + rsinfo.getServerName());
594      }
595    }
596
597    // From the master, get a list of all dead region servers
598    Collection<ServerName> deadRegionServers = status.getDeadServerNames();
599    errors.print("Number of dead region servers: " + deadRegionServers.size());
600    if (details) {
601      for (ServerName name: deadRegionServers) {
602        errors.print("  " + name);
603      }
604    }
605
606    // Print the current master name and state
607    errors.print("Master: " + status.getMasterName());
608
609    // Print the list of all backup masters
610    Collection<ServerName> backupMasters = status.getBackupMasterNames();
611    errors.print("Number of backup masters: " + backupMasters.size());
612    if (details) {
613      for (ServerName name: backupMasters) {
614        errors.print("  " + name);
615      }
616    }
617
618    errors.print("Average load: " + status.getAverageLoad());
619    errors.print("Number of requests: " + status.getRequestCount());
620    errors.print("Number of regions: " + status.getRegionCount());
621
622    List<RegionState> rits = status.getRegionStatesInTransition();
623    errors.print("Number of regions in transition: " + rits.size());
624    if (details) {
625      for (RegionState state: rits) {
626        errors.print("  " + state.toDescriptiveString());
627      }
628    }
629
630    // Determine what's deployed
631    processRegionServers(regionServers);
632  }
633
634  /**
635   * Clear the current state of hbck.
636   */
637  private void clearState() {
638    // Make sure regionInfo is empty before starting
639    fixes = 0;
640    regionInfoMap.clear();
641    emptyRegionInfoQualifiers.clear();
642    tableStates.clear();
643    errors.clear();
644    tablesInfo.clear();
645    orphanHdfsDirs.clear();
646    skippedRegions.clear();
647  }
648
649  /**
650   * This repair method analyzes hbase data in hdfs and repairs it to satisfy
651   * the table integrity rules.  HBase doesn't need to be online for this
652   * operation to work.
653   */
654  public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
655    // Initial pass to fix orphans.
656    if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
657        || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
658      LOG.info("Loading regioninfos HDFS");
659      // if nothing is happening this should always complete in two iterations.
660      int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
661      int curIter = 0;
662      do {
663        clearState(); // clears hbck state and reset fixes to 0 and.
664        // repair what's on HDFS
665        restoreHdfsIntegrity();
666        curIter++;// limit the number of iterations.
667      } while (fixes > 0 && curIter <= maxIterations);
668
669      // Repairs should be done in the first iteration and verification in the second.
670      // If there are more than 2 passes, something funny has happened.
671      if (curIter > 2) {
672        if (curIter == maxIterations) {
673          LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
674              + "Tables integrity may not be fully repaired!");
675        } else {
676          LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
677        }
678      }
679    }
680  }
681
682  /**
683   * This repair method requires the cluster to be online since it contacts
684   * region servers and the masters.  It makes each region's state in HDFS, in
685   * hbase:meta, and deployments consistent.
686   *
687   * @return If &gt; 0 , number of errors detected, if &lt; 0 there was an unrecoverable
688   *     error.  If 0, we have a clean hbase.
689   */
690  public int onlineConsistencyRepair() throws IOException, KeeperException,
691    InterruptedException {
692
693    // get regions according to what is online on each RegionServer
694    loadDeployedRegions();
695    // check whether hbase:meta is deployed and online
696    recordMetaRegion();
697    // Check if hbase:meta is found only once and in the right place
698    if (!checkMetaRegion()) {
699      String errorMsg = "hbase:meta table is not consistent. ";
700      if (shouldFixAssignments()) {
701        errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
702      } else {
703        errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
704      }
705      errors.reportError(errorMsg + " Exiting...");
706      return -2;
707    }
708    // Not going with further consistency check for tables when hbase:meta itself is not consistent.
709    LOG.info("Loading regionsinfo from the hbase:meta table");
710    boolean success = loadMetaEntries();
711    if (!success) return -1;
712
713    // Empty cells in hbase:meta?
714    reportEmptyMetaCells();
715
716    // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
717    if (shouldFixEmptyMetaCells()) {
718      fixEmptyMetaCells();
719    }
720
721    // get a list of all tables that have not changed recently.
722    if (!checkMetaOnly) {
723      reportTablesInFlux();
724    }
725
726    // Get disabled tables states
727    loadTableStates();
728
729    // load regiondirs and regioninfos from HDFS
730    if (shouldCheckHdfs()) {
731      LOG.info("Loading region directories from HDFS");
732      loadHdfsRegionDirs();
733      LOG.info("Loading region information from HDFS");
734      loadHdfsRegionInfos();
735    }
736
737    // fix the orphan tables
738    fixOrphanTables();
739
740    LOG.info("Checking and fixing region consistency");
741    // Check and fix consistency
742    checkAndFixConsistency();
743
744    // Check integrity (does not fix)
745    checkIntegrity();
746    return errors.getErrorList().size();
747  }
748
749  /**
750   * This method maintains an ephemeral znode. If the creation fails we return false or throw
751   * exception
752   *
753   * @return true if creating znode succeeds; false otherwise
754   * @throws IOException if IO failure occurs
755   */
756  private boolean setMasterInMaintenanceMode() throws IOException {
757    RetryCounter retryCounter = createZNodeRetryCounterFactory.create();
758    hbckEphemeralNodePath = ZNodePaths.joinZNode(
759      zkw.getZNodePaths().masterMaintZNode,
760      "hbck-" + Long.toString(EnvironmentEdgeManager.currentTime()));
761    do {
762      try {
763        hbckZodeCreated = ZKUtil.createEphemeralNodeAndWatch(zkw, hbckEphemeralNodePath, null);
764        if (hbckZodeCreated) {
765          break;
766        }
767      } catch (KeeperException e) {
768        if (retryCounter.getAttemptTimes() >= retryCounter.getMaxAttempts()) {
769           throw new IOException("Can't create znode " + hbckEphemeralNodePath, e);
770        }
771        // fall through and retry
772      }
773
774      LOG.warn("Fail to create znode " + hbckEphemeralNodePath + ", try=" +
775          (retryCounter.getAttemptTimes() + 1) + " of " + retryCounter.getMaxAttempts());
776
777      try {
778        retryCounter.sleepUntilNextRetry();
779      } catch (InterruptedException ie) {
780        throw (InterruptedIOException) new InterruptedIOException(
781              "Can't create znode " + hbckEphemeralNodePath).initCause(ie);
782      }
783    } while (retryCounter.shouldRetry());
784    return hbckZodeCreated;
785  }
786
787  private void cleanupHbckZnode() {
788    try {
789      if (zkw != null && hbckZodeCreated) {
790        ZKUtil.deleteNode(zkw, hbckEphemeralNodePath);
791        hbckZodeCreated = false;
792      }
793    } catch (KeeperException e) {
794      // Ignore
795      if (!e.code().equals(KeeperException.Code.NONODE)) {
796        LOG.warn("Delete HBCK znode " + hbckEphemeralNodePath + " failed ", e);
797      }
798    }
799  }
800
801  /**
802   * Contacts the master and prints out cluster-wide information
803   * @return 0 on success, non-zero on failure
804   */
805  public int onlineHbck()
806      throws IOException, KeeperException, InterruptedException, ReplicationException {
807    // print hbase server version
808    errors.print("Version: " + status.getHBaseVersion());
809
810    // Clean start
811    clearState();
812    // Do offline check and repair first
813    offlineHdfsIntegrityRepair();
814    offlineReferenceFileRepair();
815    offlineHLinkFileRepair();
816    // If Master runs maintenance tasks (such as balancer, catalog janitor, etc) during online
817    // hbck, it is likely that hbck would be misled and report transient errors.  Therefore, it
818    // is better to set Master into maintenance mode during online hbck.
819    //
820    if (!setMasterInMaintenanceMode()) {
821      LOG.warn("HBCK is running while master is not in maintenance mode, you might see transient "
822        + "error.  Please run HBCK multiple times to reduce the chance of transient error.");
823    }
824
825    onlineConsistencyRepair();
826
827    if (checkRegionBoundaries) {
828      checkRegionBoundaries();
829    }
830
831    checkAndFixReplication();
832
833    cleanReplicationBarrier();
834
835    // Remove the hbck znode
836    cleanupHbckZnode();
837
838    // Remove the hbck lock
839    unlockHbck();
840
841    // Print table summary
842    printTableSummary(tablesInfo);
843    return errors.summarize();
844  }
845
846  public static byte[] keyOnly(byte[] b) {
847    if (b == null)
848      return b;
849    int rowlength = Bytes.toShort(b, 0);
850    byte[] result = new byte[rowlength];
851    System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
852    return result;
853  }
854
855  @Override
856  public void close() throws IOException {
857    try {
858      cleanupHbckZnode();
859      unlockHbck();
860    } catch (Exception io) {
861      LOG.warn(io.toString(), io);
862    } finally {
863      if (zkw != null) {
864        zkw.close();
865        zkw = null;
866      }
867      IOUtils.closeQuietly(admin);
868      IOUtils.closeQuietly(meta);
869      IOUtils.closeQuietly(connection);
870    }
871  }
872
873  private static class RegionBoundariesInformation {
874    public byte [] regionName;
875    public byte [] metaFirstKey;
876    public byte [] metaLastKey;
877    public byte [] storesFirstKey;
878    public byte [] storesLastKey;
879    @Override
880    public String toString () {
881      return "regionName=" + Bytes.toStringBinary(regionName) +
882             "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
883             "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
884             "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
885             "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
886    }
887  }
888
889  public void checkRegionBoundaries() {
890    try {
891      ByteArrayComparator comparator = new ByteArrayComparator();
892      List<RegionInfo> regions = MetaTableAccessor.getAllRegions(connection, true);
893      final RegionBoundariesInformation currentRegionBoundariesInformation =
894          new RegionBoundariesInformation();
895      Path hbaseRoot = FSUtils.getRootDir(getConf());
896      for (RegionInfo regionInfo : regions) {
897        Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
898        currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
899        // For each region, get the start and stop key from the META and compare them to the
900        // same information from the Stores.
901        Path path = new Path(tableDir, regionInfo.getEncodedName());
902        FileSystem fs = path.getFileSystem(getConf());
903        FileStatus[] files = fs.listStatus(path);
904        // For all the column families in this region...
905        byte[] storeFirstKey = null;
906        byte[] storeLastKey = null;
907        for (FileStatus file : files) {
908          String fileName = file.getPath().toString();
909          fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
910          if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
911            FileStatus[] storeFiles = fs.listStatus(file.getPath());
912            // For all the stores in this column family.
913            for (FileStatus storeFile : storeFiles) {
914              HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(),
915                CacheConfig.DISABLED, true, getConf());
916              if ((reader.getFirstKey() != null)
917                  && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
918                      ((KeyValue.KeyOnlyKeyValue) reader.getFirstKey().get()).getKey()) > 0))) {
919                storeFirstKey = ((KeyValue.KeyOnlyKeyValue)reader.getFirstKey().get()).getKey();
920              }
921              if ((reader.getLastKey() != null)
922                  && ((storeLastKey == null) || (comparator.compare(storeLastKey,
923                      ((KeyValue.KeyOnlyKeyValue)reader.getLastKey().get()).getKey())) < 0)) {
924                storeLastKey = ((KeyValue.KeyOnlyKeyValue)reader.getLastKey().get()).getKey();
925              }
926              reader.close();
927            }
928          }
929        }
930        currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
931        currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
932        currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
933        currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
934        if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
935          currentRegionBoundariesInformation.metaFirstKey = null;
936        if (currentRegionBoundariesInformation.metaLastKey.length == 0)
937          currentRegionBoundariesInformation.metaLastKey = null;
938
939        // For a region to be correct, we need the META start key to be smaller or equal to the
940        // smallest start key from all the stores, and the start key from the next META entry to
941        // be bigger than the last key from all the current stores. First region start key is null;
942        // Last region end key is null; some regions can be empty and not have any store.
943
944        boolean valid = true;
945        // Checking start key.
946        if ((currentRegionBoundariesInformation.storesFirstKey != null)
947            && (currentRegionBoundariesInformation.metaFirstKey != null)) {
948          valid = valid
949              && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
950                currentRegionBoundariesInformation.metaFirstKey) >= 0;
951        }
952        // Checking stop key.
953        if ((currentRegionBoundariesInformation.storesLastKey != null)
954            && (currentRegionBoundariesInformation.metaLastKey != null)) {
955          valid = valid
956              && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
957                currentRegionBoundariesInformation.metaLastKey) < 0;
958        }
959        if (!valid) {
960          errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
961            tablesInfo.get(regionInfo.getTable()));
962          LOG.warn("Region's boundaries not aligned between stores and META for:");
963          LOG.warn(Objects.toString(currentRegionBoundariesInformation));
964        }
965      }
966    } catch (IOException e) {
967      LOG.error(e.toString(), e);
968    }
969  }
970
971  /**
972   * Iterates through the list of all orphan/invalid regiondirs.
973   */
974  private void adoptHdfsOrphans(Collection<HbckRegionInfo> orphanHdfsDirs) throws IOException {
975    for (HbckRegionInfo hi : orphanHdfsDirs) {
976      LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
977      adoptHdfsOrphan(hi);
978    }
979  }
980
981  /**
982   * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
983   * these orphans by creating a new region, and moving the column families,
984   * recovered edits, WALs, into the new region dir.  We determine the region
985   * startkey and endkeys by looking at all of the hfiles inside the column
986   * families to identify the min and max keys. The resulting region will
987   * likely violate table integrity but will be dealt with by merging
988   * overlapping regions.
989   */
990  @SuppressWarnings("deprecation")
991  private void adoptHdfsOrphan(HbckRegionInfo hi) throws IOException {
992    Path p = hi.getHdfsRegionDir();
993    FileSystem fs = p.getFileSystem(getConf());
994    FileStatus[] dirs = fs.listStatus(p);
995    if (dirs == null) {
996      LOG.warn("Attempt to adopt orphan hdfs region skipped because no files present in " +
997          p + ". This dir could probably be deleted.");
998      return ;
999    }
1000
1001    TableName tableName = hi.getTableName();
1002    HbckTableInfo tableInfo = tablesInfo.get(tableName);
1003    Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
1004    TableDescriptor template = tableInfo.getTableDescriptor();
1005
1006    // find min and max key values
1007    Pair<byte[],byte[]> orphanRegionRange = null;
1008    for (FileStatus cf : dirs) {
1009      String cfName= cf.getPath().getName();
1010      // TODO Figure out what the special dirs are
1011      if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
1012
1013      FileStatus[] hfiles = fs.listStatus(cf.getPath());
1014      for (FileStatus hfile : hfiles) {
1015        byte[] start, end;
1016        HFile.Reader hf = null;
1017        try {
1018          hf = HFile.createReader(fs, hfile.getPath(), CacheConfig.DISABLED, true, getConf());
1019          hf.loadFileInfo();
1020          Optional<Cell> startKv = hf.getFirstKey();
1021          start = CellUtil.cloneRow(startKv.get());
1022          Optional<Cell> endKv = hf.getLastKey();
1023          end = CellUtil.cloneRow(endKv.get());
1024        } catch (IOException ioe) {
1025          LOG.warn("Problem reading orphan file " + hfile + ", skipping");
1026          continue;
1027        } catch (NullPointerException ioe) {
1028          LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
1029          continue;
1030        } finally {
1031          if (hf != null) {
1032            hf.close();
1033          }
1034        }
1035
1036        // expand the range to include the range of all hfiles
1037        if (orphanRegionRange == null) {
1038          // first range
1039          orphanRegionRange = new Pair<>(start, end);
1040        } else {
1041          // TODO add test
1042
1043          // expand range only if the hfile is wider.
1044          if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
1045            orphanRegionRange.setFirst(start);
1046          }
1047          if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
1048            orphanRegionRange.setSecond(end);
1049          }
1050        }
1051      }
1052    }
1053    if (orphanRegionRange == null) {
1054      LOG.warn("No data in dir " + p + ", sidelining data");
1055      fixes++;
1056      sidelineRegionDir(fs, hi);
1057      return;
1058    }
1059    LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
1060        Bytes.toString(orphanRegionRange.getSecond()) + ")");
1061
1062    // create new region on hdfs. move data into place.
1063    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(template.getTableName())
1064        .setStartKey(orphanRegionRange.getFirst())
1065        .setEndKey(Bytes.add(orphanRegionRange.getSecond(), new byte[1]))
1066        .build();
1067    LOG.info("Creating new region : " + regionInfo);
1068    HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), regionInfo, template);
1069    Path target = region.getRegionFileSystem().getRegionDir();
1070
1071    // rename all the data to new region
1072    mergeRegionDirs(target, hi);
1073    fixes++;
1074  }
1075
1076  /**
1077   * This method determines if there are table integrity errors in HDFS.  If
1078   * there are errors and the appropriate "fix" options are enabled, the method
1079   * will first correct orphan regions making them into legit regiondirs, and
1080   * then reload to merge potentially overlapping regions.
1081   *
1082   * @return number of table integrity errors found
1083   */
1084  private int restoreHdfsIntegrity() throws IOException, InterruptedException {
1085    // Determine what's on HDFS
1086    LOG.info("Loading HBase regioninfo from HDFS...");
1087    loadHdfsRegionDirs(); // populating regioninfo table.
1088
1089    int errs = errors.getErrorList().size();
1090    // First time just get suggestions.
1091    tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1092    checkHdfsIntegrity(false, false);
1093
1094    if (errors.getErrorList().size() == errs) {
1095      LOG.info("No integrity errors.  We are done with this phase. Glorious.");
1096      return 0;
1097    }
1098
1099    if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
1100      adoptHdfsOrphans(orphanHdfsDirs);
1101      // TODO optimize by incrementally adding instead of reloading.
1102    }
1103
1104    // Make sure there are no holes now.
1105    if (shouldFixHdfsHoles()) {
1106      clearState(); // this also resets # fixes.
1107      loadHdfsRegionDirs();
1108      tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1109      tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
1110    }
1111
1112    // Now we fix overlaps
1113    if (shouldFixHdfsOverlaps()) {
1114      // second pass we fix overlaps.
1115      clearState(); // this also resets # fixes.
1116      loadHdfsRegionDirs();
1117      tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1118      tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
1119    }
1120
1121    return errors.getErrorList().size();
1122  }
1123
1124  /**
1125   * Scan all the store file names to find any lingering reference files,
1126   * which refer to some none-exiting files. If "fix" option is enabled,
1127   * any lingering reference file will be sidelined if found.
1128   * <p>
1129   * Lingering reference file prevents a region from opening. It has to
1130   * be fixed before a cluster can start properly.
1131   */
1132  private void offlineReferenceFileRepair() throws IOException, InterruptedException {
1133    clearState();
1134    Configuration conf = getConf();
1135    Path hbaseRoot = FSUtils.getRootDir(conf);
1136    FileSystem fs = hbaseRoot.getFileSystem(conf);
1137    LOG.info("Computing mapping of all store files");
1138    Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot,
1139      new FSUtils.ReferenceFileFilter(fs), executor, errors);
1140    errors.print("");
1141    LOG.info("Validating mapping using HDFS state");
1142    for (Path path: allFiles.values()) {
1143      Path referredToFile = StoreFileInfo.getReferredToFile(path);
1144      if (fs.exists(referredToFile)) continue;  // good, expected
1145
1146      // Found a lingering reference file
1147      errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
1148        "Found lingering reference file " + path);
1149      if (!shouldFixReferenceFiles()) continue;
1150
1151      // Now, trying to fix it since requested
1152      boolean success = false;
1153      String pathStr = path.toString();
1154
1155      // A reference file path should be like
1156      // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
1157      // Up 5 directories to get the root folder.
1158      // So the file will be sidelined to a similar folder structure.
1159      int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
1160      for (int i = 0; index > 0 && i < 5; i++) {
1161        index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
1162      }
1163      if (index > 0) {
1164        Path rootDir = getSidelineDir();
1165        Path dst = new Path(rootDir, pathStr.substring(index + 1));
1166        fs.mkdirs(dst.getParent());
1167        LOG.info("Trying to sideline reference file "
1168          + path + " to " + dst);
1169        setShouldRerun();
1170
1171        success = fs.rename(path, dst);
1172        debugLsr(dst);
1173
1174      }
1175      if (!success) {
1176        LOG.error("Failed to sideline reference file " + path);
1177      }
1178    }
1179  }
1180
1181  /**
1182   * Scan all the store file names to find any lingering HFileLink files,
1183   * which refer to some none-exiting files. If "fix" option is enabled,
1184   * any lingering HFileLink file will be sidelined if found.
1185   */
1186  private void offlineHLinkFileRepair() throws IOException, InterruptedException {
1187    Configuration conf = getConf();
1188    Path hbaseRoot = FSUtils.getRootDir(conf);
1189    FileSystem fs = hbaseRoot.getFileSystem(conf);
1190    LOG.info("Computing mapping of all link files");
1191    Map<String, Path> allFiles = FSUtils
1192        .getTableStoreFilePathMap(fs, hbaseRoot, new FSUtils.HFileLinkFilter(), executor, errors);
1193    errors.print("");
1194
1195    LOG.info("Validating mapping using HDFS state");
1196    for (Path path : allFiles.values()) {
1197      // building HFileLink object to gather locations
1198      HFileLink actualLink = HFileLink.buildFromHFileLinkPattern(conf, path);
1199      if (actualLink.exists(fs)) continue; // good, expected
1200
1201      // Found a lingering HFileLink
1202      errors.reportError(ERROR_CODE.LINGERING_HFILELINK, "Found lingering HFileLink " + path);
1203      if (!shouldFixHFileLinks()) continue;
1204
1205      // Now, trying to fix it since requested
1206      setShouldRerun();
1207
1208      // An HFileLink path should be like
1209      // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/linkedtable=linkedregionname-linkedhfilename
1210      // sidelineing will happen in the ${hbase.rootdir}/${sidelinedir} directory with the same folder structure.
1211      boolean success = sidelineFile(fs, hbaseRoot, path);
1212
1213      if (!success) {
1214        LOG.error("Failed to sideline HFileLink file " + path);
1215      }
1216
1217      // An HFileLink backreference path should be like
1218      // ${hbase.rootdir}/archive/data/namespace/table_name/region_id/family_name/.links-linkedhfilename
1219      // sidelineing will happen in the ${hbase.rootdir}/${sidelinedir} directory with the same folder structure.
1220      Path backRefPath = FileLink.getBackReferencesDir(HFileArchiveUtil
1221              .getStoreArchivePath(conf, HFileLink.getReferencedTableName(path.getName().toString()),
1222                  HFileLink.getReferencedRegionName(path.getName().toString()),
1223                  path.getParent().getName()),
1224          HFileLink.getReferencedHFileName(path.getName().toString()));
1225      success = sidelineFile(fs, hbaseRoot, backRefPath);
1226
1227      if (!success) {
1228        LOG.error("Failed to sideline HFileLink backreference file " + path);
1229      }
1230    }
1231  }
1232
1233  private boolean sidelineFile(FileSystem fs, Path hbaseRoot, Path path) throws IOException {
1234    URI uri = hbaseRoot.toUri().relativize(path.toUri());
1235    if (uri.isAbsolute()) return false;
1236    String relativePath = uri.getPath();
1237    Path rootDir = getSidelineDir();
1238    Path dst = new Path(rootDir, relativePath);
1239    boolean pathCreated = fs.mkdirs(dst.getParent());
1240    if (!pathCreated) {
1241      LOG.error("Failed to create path: " + dst.getParent());
1242      return false;
1243    }
1244    LOG.info("Trying to sideline file " + path + " to " + dst);
1245    return fs.rename(path, dst);
1246  }
1247
1248  /**
1249   * TODO -- need to add tests for this.
1250   */
1251  private void reportEmptyMetaCells() {
1252    errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
1253      emptyRegionInfoQualifiers.size());
1254    if (details) {
1255      for (Result r: emptyRegionInfoQualifiers) {
1256        errors.print("  " + r);
1257      }
1258    }
1259  }
1260
1261  /**
1262   * TODO -- need to add tests for this.
1263   */
1264  private void reportTablesInFlux() {
1265    AtomicInteger numSkipped = new AtomicInteger(0);
1266    TableDescriptor[] allTables = getTables(numSkipped);
1267    errors.print("Number of Tables: " + allTables.length);
1268    if (details) {
1269      if (numSkipped.get() > 0) {
1270        errors.detail("Number of Tables in flux: " + numSkipped.get());
1271      }
1272      for (TableDescriptor td : allTables) {
1273        errors.detail("  Table: " + td.getTableName() + "\t" +
1274                           (td.isReadOnly() ? "ro" : "rw") + "\t" +
1275                            (td.isMetaRegion() ? "META" : "    ") + "\t" +
1276                           " families: " + td.getColumnFamilyCount());
1277      }
1278    }
1279  }
1280
1281  public HbckErrorReporter getErrors() {
1282    return errors;
1283  }
1284
1285  /**
1286   * Populate hbi's from regionInfos loaded from file system.
1287   */
1288  private SortedMap<TableName, HbckTableInfo> loadHdfsRegionInfos()
1289      throws IOException, InterruptedException {
1290    tablesInfo.clear(); // regenerating the data
1291    // generate region split structure
1292    Collection<HbckRegionInfo> hbckRegionInfos = regionInfoMap.values();
1293
1294    // Parallelized read of .regioninfo files.
1295    List<WorkItemHdfsRegionInfo> hbis = new ArrayList<>(hbckRegionInfos.size());
1296    List<Future<Void>> hbiFutures;
1297
1298    for (HbckRegionInfo hbi : hbckRegionInfos) {
1299      WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1300      hbis.add(work);
1301    }
1302
1303    // Submit and wait for completion
1304    hbiFutures = executor.invokeAll(hbis);
1305
1306    for(int i=0; i<hbiFutures.size(); i++) {
1307      WorkItemHdfsRegionInfo work = hbis.get(i);
1308      Future<Void> f = hbiFutures.get(i);
1309      try {
1310        f.get();
1311      } catch(ExecutionException e) {
1312        LOG.warn("Failed to read .regioninfo file for region " +
1313              work.hbi.getRegionNameAsString(), e.getCause());
1314      }
1315    }
1316
1317    Path hbaseRoot = FSUtils.getRootDir(getConf());
1318    FileSystem fs = hbaseRoot.getFileSystem(getConf());
1319    // serialized table info gathering.
1320    for (HbckRegionInfo hbi: hbckRegionInfos) {
1321
1322      if (hbi.getHdfsHRI() == null) {
1323        // was an orphan
1324        continue;
1325      }
1326
1327
1328      // get table name from hdfs, populate various HBaseFsck tables.
1329      TableName tableName = hbi.getTableName();
1330      if (tableName == null) {
1331        // There was an entry in hbase:meta not in the HDFS?
1332        LOG.warn("tableName was null for: " + hbi);
1333        continue;
1334      }
1335
1336      HbckTableInfo modTInfo = tablesInfo.get(tableName);
1337      if (modTInfo == null) {
1338        // only executed once per table.
1339        modTInfo = new HbckTableInfo(tableName, this);
1340        tablesInfo.put(tableName, modTInfo);
1341        try {
1342          TableDescriptor htd =
1343              FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1344          modTInfo.htds.add(htd);
1345        } catch (IOException ioe) {
1346          if (!orphanTableDirs.containsKey(tableName)) {
1347            LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1348            //should only report once for each table
1349            errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1350                "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1351            Set<String> columns = new HashSet<>();
1352            orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1353          }
1354        }
1355      }
1356      if (!hbi.isSkipChecks()) {
1357        modTInfo.addRegionInfo(hbi);
1358      }
1359    }
1360
1361    loadTableInfosForTablesWithNoRegion();
1362    errors.print("");
1363
1364    return tablesInfo;
1365  }
1366
1367  /**
1368   * To get the column family list according to the column family dirs
1369   * @param columns
1370   * @param hbi
1371   * @return a set of column families
1372   * @throws IOException
1373   */
1374  private Set<String> getColumnFamilyList(Set<String> columns, HbckRegionInfo hbi)
1375      throws IOException {
1376    Path regionDir = hbi.getHdfsRegionDir();
1377    FileSystem fs = regionDir.getFileSystem(getConf());
1378    FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1379    for (FileStatus subdir : subDirs) {
1380      String columnfamily = subdir.getPath().getName();
1381      columns.add(columnfamily);
1382    }
1383    return columns;
1384  }
1385
1386  /**
1387   * To fabricate a .tableinfo file with following contents<br>
1388   * 1. the correct tablename <br>
1389   * 2. the correct colfamily list<br>
1390   * 3. the default properties for both {@link TableDescriptor} and {@link ColumnFamilyDescriptor}<br>
1391   * @throws IOException
1392   */
1393  private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1394      Set<String> columns) throws IOException {
1395    if (columns ==null || columns.isEmpty()) return false;
1396    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
1397    for (String columnfamimly : columns) {
1398      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(columnfamimly));
1399    }
1400    fstd.createTableDescriptor(builder.build(), true);
1401    return true;
1402  }
1403
1404  /**
1405   * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1406   * @throws IOException
1407   */
1408  public void fixEmptyMetaCells() throws IOException {
1409    if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1410      LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1411      for (Result region : emptyRegionInfoQualifiers) {
1412        deleteMetaRegion(region.getRow());
1413        errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1414      }
1415      emptyRegionInfoQualifiers.clear();
1416    }
1417  }
1418
1419  /**
1420   * To fix orphan table by creating a .tableinfo file under tableDir <br>
1421   * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1422   * 2. else create a default .tableinfo file with following items<br>
1423   * &nbsp;2.1 the correct tablename <br>
1424   * &nbsp;2.2 the correct colfamily list<br>
1425   * &nbsp;2.3 the default properties for both {@link TableDescriptor} and {@link ColumnFamilyDescriptor}<br>
1426   * @throws IOException
1427   */
1428  public void fixOrphanTables() throws IOException {
1429    if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1430
1431      List<TableName> tmpList = new ArrayList<>(orphanTableDirs.keySet().size());
1432      tmpList.addAll(orphanTableDirs.keySet());
1433      TableDescriptor[] htds = getTableDescriptors(tmpList);
1434      Iterator<Entry<TableName, Set<String>>> iter =
1435          orphanTableDirs.entrySet().iterator();
1436      int j = 0;
1437      int numFailedCase = 0;
1438      FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1439      while (iter.hasNext()) {
1440        Entry<TableName, Set<String>> entry =
1441            iter.next();
1442        TableName tableName = entry.getKey();
1443        LOG.info("Trying to fix orphan table error: " + tableName);
1444        if (j < htds.length) {
1445          if (tableName.equals(htds[j].getTableName())) {
1446            TableDescriptor htd = htds[j];
1447            LOG.info("fixing orphan table: " + tableName + " from cache");
1448            fstd.createTableDescriptor(htd, true);
1449            j++;
1450            iter.remove();
1451          }
1452        } else {
1453          if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1454            LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1455            LOG.warn("Strongly recommend to modify the TableDescriptor if necessary for: " + tableName);
1456            iter.remove();
1457          } else {
1458            LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1459            numFailedCase++;
1460          }
1461        }
1462        fixes++;
1463      }
1464
1465      if (orphanTableDirs.isEmpty()) {
1466        // all orphanTableDirs are luckily recovered
1467        // re-run doFsck after recovering the .tableinfo file
1468        setShouldRerun();
1469        LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1470      } else if (numFailedCase > 0) {
1471        LOG.error("Failed to fix " + numFailedCase
1472            + " OrphanTables with default .tableinfo files");
1473      }
1474
1475    }
1476    //cleanup the list
1477    orphanTableDirs.clear();
1478
1479  }
1480
1481  /**
1482   * Log an appropriate message about whether or not overlapping merges are computed in parallel.
1483   */
1484  private void logParallelMerge() {
1485    if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
1486      LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
1487          " false to run serially.");
1488    } else {
1489      LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
1490          " true to run in parallel.");
1491    }
1492  }
1493
1494  private SortedMap<TableName, HbckTableInfo> checkHdfsIntegrity(boolean fixHoles,
1495      boolean fixOverlaps) throws IOException {
1496    LOG.info("Checking HBase region split map from HDFS data...");
1497    logParallelMerge();
1498    for (HbckTableInfo tInfo : tablesInfo.values()) {
1499      TableIntegrityErrorHandler handler;
1500      if (fixHoles || fixOverlaps) {
1501        handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1502          fixHoles, fixOverlaps);
1503      } else {
1504        handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1505      }
1506      if (!tInfo.checkRegionChain(handler)) {
1507        // should dump info as well.
1508        errors.report("Found inconsistency in table " + tInfo.getName());
1509      }
1510    }
1511    return tablesInfo;
1512  }
1513
1514  Path getSidelineDir() throws IOException {
1515    if (sidelineDir == null) {
1516      Path hbaseDir = FSUtils.getRootDir(getConf());
1517      Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1518      sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1519          + startMillis);
1520    }
1521    return sidelineDir;
1522  }
1523
1524  /**
1525   * Sideline a region dir (instead of deleting it)
1526   */
1527  Path sidelineRegionDir(FileSystem fs, HbckRegionInfo hi) throws IOException {
1528    return sidelineRegionDir(fs, null, hi);
1529  }
1530
1531  /**
1532   * Sideline a region dir (instead of deleting it)
1533   *
1534   * @param parentDir if specified, the region will be sidelined to folder like
1535   *     {@literal .../parentDir/<table name>/<region name>}. The purpose is to group together
1536   *     similar regions sidelined, for example, those regions should be bulk loaded back later
1537   *     on. If NULL, it is ignored.
1538   */
1539  Path sidelineRegionDir(FileSystem fs,
1540      String parentDir, HbckRegionInfo hi) throws IOException {
1541    TableName tableName = hi.getTableName();
1542    Path regionDir = hi.getHdfsRegionDir();
1543
1544    if (!fs.exists(regionDir)) {
1545      LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1546      return null;
1547    }
1548
1549    Path rootDir = getSidelineDir();
1550    if (parentDir != null) {
1551      rootDir = new Path(rootDir, parentDir);
1552    }
1553    Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1554    Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1555    fs.mkdirs(sidelineRegionDir);
1556    boolean success = false;
1557    FileStatus[] cfs =  fs.listStatus(regionDir);
1558    if (cfs == null) {
1559      LOG.info("Region dir is empty: " + regionDir);
1560    } else {
1561      for (FileStatus cf : cfs) {
1562        Path src = cf.getPath();
1563        Path dst =  new Path(sidelineRegionDir, src.getName());
1564        if (fs.isFile(src)) {
1565          // simple file
1566          success = fs.rename(src, dst);
1567          if (!success) {
1568            String msg = "Unable to rename file " + src +  " to " + dst;
1569            LOG.error(msg);
1570            throw new IOException(msg);
1571          }
1572          continue;
1573        }
1574
1575        // is a directory.
1576        fs.mkdirs(dst);
1577
1578        LOG.info("Sidelining files from " + src + " into containing region " + dst);
1579        // FileSystem.rename is inconsistent with directories -- if the
1580        // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1581        // it moves the src into the dst dir resulting in (foo/a/b).  If
1582        // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1583        FileStatus[] hfiles = fs.listStatus(src);
1584        if (hfiles != null && hfiles.length > 0) {
1585          for (FileStatus hfile : hfiles) {
1586            success = fs.rename(hfile.getPath(), dst);
1587            if (!success) {
1588              String msg = "Unable to rename file " + src +  " to " + dst;
1589              LOG.error(msg);
1590              throw new IOException(msg);
1591            }
1592          }
1593        }
1594        LOG.debug("Sideline directory contents:");
1595        debugLsr(sidelineRegionDir);
1596      }
1597    }
1598
1599    LOG.info("Removing old region dir: " + regionDir);
1600    success = fs.delete(regionDir, true);
1601    if (!success) {
1602      String msg = "Unable to delete dir " + regionDir;
1603      LOG.error(msg);
1604      throw new IOException(msg);
1605    }
1606    return sidelineRegionDir;
1607  }
1608
1609  /**
1610   * Load the list of disabled tables in ZK into local set.
1611   * @throws ZooKeeperConnectionException
1612   * @throws IOException
1613   */
1614  private void loadTableStates()
1615  throws IOException {
1616    tableStates = MetaTableAccessor.getTableStates(connection);
1617    // Add hbase:meta so this tool keeps working. In hbase2, meta is always enabled though it
1618    // has no entry in the table states. HBCK doesn't work right w/ hbase2 but just do this in
1619    // meantime.
1620    this.tableStates.put(TableName.META_TABLE_NAME,
1621        new TableState(TableName.META_TABLE_NAME, TableState.State.ENABLED));
1622  }
1623
1624  /**
1625   * Check if the specified region's table is disabled.
1626   * @param tableName table to check status of
1627   */
1628  boolean isTableDisabled(TableName tableName) {
1629    return tableStates.containsKey(tableName)
1630        && tableStates.get(tableName)
1631        .inStates(TableState.State.DISABLED, TableState.State.DISABLING);
1632  }
1633
1634  /**
1635   * Scan HDFS for all regions, recording their information into
1636   * regionInfoMap
1637   */
1638  public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1639    Path rootDir = FSUtils.getRootDir(getConf());
1640    FileSystem fs = rootDir.getFileSystem(getConf());
1641
1642    // list all tables from HDFS
1643    List<FileStatus> tableDirs = Lists.newArrayList();
1644
1645    boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1646
1647    List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1648    for (Path path : paths) {
1649      TableName tableName = FSUtils.getTableName(path);
1650       if ((!checkMetaOnly &&
1651           isTableIncluded(tableName)) ||
1652           tableName.equals(TableName.META_TABLE_NAME)) {
1653         tableDirs.add(fs.getFileStatus(path));
1654       }
1655    }
1656
1657    // verify that version file exists
1658    if (!foundVersionFile) {
1659      errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1660          "Version file does not exist in root dir " + rootDir);
1661      if (shouldFixVersionFile()) {
1662        LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1663            + " file.");
1664        setShouldRerun();
1665        FSUtils.setVersion(fs, rootDir, getConf().getInt(
1666            HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1667            HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1668            HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1669      }
1670    }
1671
1672    // Avoid multithreading at table-level because already multithreaded internally at
1673    // region-level.  Additionally multithreading at table-level can lead to deadlock
1674    // if there are many tables in the cluster.  Since there are a limited # of threads
1675    // in the executor's thread pool and if we multithread at the table-level by putting
1676    // WorkItemHdfsDir callables into the executor, then we will have some threads in the
1677    // executor tied up solely in waiting for the tables' region-level calls to complete.
1678    // If there are enough tables then there will be no actual threads in the pool left
1679    // for the region-level callables to be serviced.
1680    for (FileStatus tableDir : tableDirs) {
1681      LOG.debug("Loading region dirs from " +tableDir.getPath());
1682      WorkItemHdfsDir item = new WorkItemHdfsDir(fs, errors, tableDir);
1683      try {
1684        item.call();
1685      } catch (ExecutionException e) {
1686        LOG.warn("Could not completely load table dir " +
1687            tableDir.getPath(), e.getCause());
1688      }
1689    }
1690    errors.print("");
1691  }
1692
1693  /**
1694   * Record the location of the hbase:meta region as found in ZooKeeper.
1695   */
1696  private boolean recordMetaRegion() throws IOException {
1697    RegionLocations rl = connection.locateRegion(TableName.META_TABLE_NAME,
1698        HConstants.EMPTY_START_ROW, false, false);
1699    if (rl == null) {
1700      errors.reportError(ERROR_CODE.NULL_META_REGION,
1701          "META region was not found in ZooKeeper");
1702      return false;
1703    }
1704    for (HRegionLocation metaLocation : rl.getRegionLocations()) {
1705      // Check if Meta region is valid and existing
1706      if (metaLocation == null ) {
1707        errors.reportError(ERROR_CODE.NULL_META_REGION,
1708            "META region location is null");
1709        return false;
1710      }
1711      if (metaLocation.getRegionInfo() == null) {
1712        errors.reportError(ERROR_CODE.NULL_META_REGION,
1713            "META location regionInfo is null");
1714        return false;
1715      }
1716      if (metaLocation.getHostname() == null) {
1717        errors.reportError(ERROR_CODE.NULL_META_REGION,
1718            "META location hostName is null");
1719        return false;
1720      }
1721      ServerName sn = metaLocation.getServerName();
1722      HbckRegionInfo.MetaEntry m = new HbckRegionInfo.MetaEntry(metaLocation.getRegion(), sn,
1723          EnvironmentEdgeManager.currentTime());
1724      HbckRegionInfo hbckRegionInfo = regionInfoMap.get(metaLocation.getRegion().getEncodedName());
1725      if (hbckRegionInfo == null) {
1726        regionInfoMap.put(metaLocation.getRegion().getEncodedName(), new HbckRegionInfo(m));
1727      } else {
1728        hbckRegionInfo.setMetaEntry(m);
1729      }
1730    }
1731    return true;
1732  }
1733
1734  private ZKWatcher createZooKeeperWatcher() throws IOException {
1735    return new ZKWatcher(getConf(), "hbase Fsck", new Abortable() {
1736      @Override
1737      public void abort(String why, Throwable e) {
1738        LOG.error(why, e);
1739        System.exit(1);
1740      }
1741
1742      @Override
1743      public boolean isAborted() {
1744        return false;
1745      }
1746
1747    });
1748  }
1749
1750  /**
1751   * Contacts each regionserver and fetches metadata about regions.
1752   * @param regionServerList - the list of region servers to connect to
1753   * @throws IOException if a remote or network exception occurs
1754   */
1755  void processRegionServers(Collection<ServerName> regionServerList)
1756    throws IOException, InterruptedException {
1757
1758    List<WorkItemRegion> workItems = new ArrayList<>(regionServerList.size());
1759    List<Future<Void>> workFutures;
1760
1761    // loop to contact each region server in parallel
1762    for (ServerName rsinfo: regionServerList) {
1763      workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1764    }
1765
1766    workFutures = executor.invokeAll(workItems);
1767
1768    for(int i=0; i<workFutures.size(); i++) {
1769      WorkItemRegion item = workItems.get(i);
1770      Future<Void> f = workFutures.get(i);
1771      try {
1772        f.get();
1773      } catch(ExecutionException e) {
1774        LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1775            e.getCause());
1776      }
1777    }
1778  }
1779
1780  /**
1781   * Check consistency of all regions that have been found in previous phases.
1782   */
1783  private void checkAndFixConsistency()
1784  throws IOException, KeeperException, InterruptedException {
1785    // Divide the checks in two phases. One for default/primary replicas and another
1786    // for the non-primary ones. Keeps code cleaner this way.
1787
1788    List<CheckRegionConsistencyWorkItem> workItems = new ArrayList<>(regionInfoMap.size());
1789    for (java.util.Map.Entry<String, HbckRegionInfo> e: regionInfoMap.entrySet()) {
1790      if (e.getValue().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
1791        workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1792      }
1793    }
1794    checkRegionConsistencyConcurrently(workItems);
1795
1796    boolean prevHdfsCheck = shouldCheckHdfs();
1797    setCheckHdfs(false); //replicas don't have any hdfs data
1798    // Run a pass over the replicas and fix any assignment issues that exist on the currently
1799    // deployed/undeployed replicas.
1800    List<CheckRegionConsistencyWorkItem> replicaWorkItems = new ArrayList<>(regionInfoMap.size());
1801    for (java.util.Map.Entry<String, HbckRegionInfo> e: regionInfoMap.entrySet()) {
1802      if (e.getValue().getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
1803        replicaWorkItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1804      }
1805    }
1806    checkRegionConsistencyConcurrently(replicaWorkItems);
1807    setCheckHdfs(prevHdfsCheck);
1808
1809    // If some regions is skipped during checkRegionConsistencyConcurrently() phase, we might
1810    // not get accurate state of the hbase if continuing. The config here allows users to tune
1811    // the tolerance of number of skipped region.
1812    // TODO: evaluate the consequence to continue the hbck operation without config.
1813    int terminateThreshold =  getConf().getInt("hbase.hbck.skipped.regions.limit", 0);
1814    int numOfSkippedRegions = skippedRegions.size();
1815    if (numOfSkippedRegions > 0 && numOfSkippedRegions > terminateThreshold) {
1816      throw new IOException(numOfSkippedRegions
1817        + " region(s) could not be checked or repaired.  See logs for detail.");
1818    }
1819
1820    if (shouldCheckHdfs()) {
1821      checkAndFixTableStates();
1822    }
1823  }
1824
1825  /**
1826   * Check consistency of all regions using mulitple threads concurrently.
1827   */
1828  private void checkRegionConsistencyConcurrently(
1829    final List<CheckRegionConsistencyWorkItem> workItems)
1830    throws IOException, KeeperException, InterruptedException {
1831    if (workItems.isEmpty()) {
1832      return;  // nothing to check
1833    }
1834
1835    List<Future<Void>> workFutures = executor.invokeAll(workItems);
1836    for(Future<Void> f: workFutures) {
1837      try {
1838        f.get();
1839      } catch(ExecutionException e1) {
1840        LOG.warn("Could not check region consistency " , e1.getCause());
1841        if (e1.getCause() instanceof IOException) {
1842          throw (IOException)e1.getCause();
1843        } else if (e1.getCause() instanceof KeeperException) {
1844          throw (KeeperException)e1.getCause();
1845        } else if (e1.getCause() instanceof InterruptedException) {
1846          throw (InterruptedException)e1.getCause();
1847        } else {
1848          throw new IOException(e1.getCause());
1849        }
1850      }
1851    }
1852  }
1853
1854  class CheckRegionConsistencyWorkItem implements Callable<Void> {
1855    private final String key;
1856    private final HbckRegionInfo hbi;
1857
1858    CheckRegionConsistencyWorkItem(String key, HbckRegionInfo hbi) {
1859      this.key = key;
1860      this.hbi = hbi;
1861    }
1862
1863    @Override
1864    public synchronized Void call() throws Exception {
1865      try {
1866        checkRegionConsistency(key, hbi);
1867      } catch (Exception e) {
1868        // If the region is non-META region, skip this region and send warning/error message; if
1869        // the region is META region, we should not continue.
1870        LOG.warn("Unable to complete check or repair the region '" + hbi.getRegionNameAsString()
1871          + "'.", e);
1872        if (hbi.getHdfsHRI().isMetaRegion()) {
1873          throw e;
1874        }
1875        LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'");
1876        addSkippedRegion(hbi);
1877      }
1878      return null;
1879    }
1880  }
1881
1882  private void addSkippedRegion(final HbckRegionInfo hbi) {
1883    Set<String> skippedRegionNames = skippedRegions.get(hbi.getTableName());
1884    if (skippedRegionNames == null) {
1885      skippedRegionNames = new HashSet<>();
1886    }
1887    skippedRegionNames.add(hbi.getRegionNameAsString());
1888    skippedRegions.put(hbi.getTableName(), skippedRegionNames);
1889  }
1890
1891  /**
1892   * Check and fix table states, assumes full info available:
1893   * - tableInfos
1894   * - empty tables loaded
1895   */
1896  private void checkAndFixTableStates() throws IOException {
1897    // first check dangling states
1898    for (Entry<TableName, TableState> entry : tableStates.entrySet()) {
1899      TableName tableName = entry.getKey();
1900      TableState tableState = entry.getValue();
1901      HbckTableInfo tableInfo = tablesInfo.get(tableName);
1902      if (isTableIncluded(tableName)
1903          && !tableName.isSystemTable()
1904          && tableInfo == null) {
1905        if (fixMeta) {
1906          MetaTableAccessor.deleteTableState(connection, tableName);
1907          TableState state = MetaTableAccessor.getTableState(connection, tableName);
1908          if (state != null) {
1909            errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,
1910                tableName + " unable to delete dangling table state " + tableState);
1911          }
1912        } else if (!checkMetaOnly) {
1913          // dangling table state in meta if checkMetaOnly is false. If checkMetaOnly is
1914          // true, tableInfo will be null as tablesInfo are not polulated for all tables from hdfs
1915          errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,
1916              tableName + " has dangling table state " + tableState);
1917        }
1918      }
1919    }
1920    // check that all tables have states
1921    for (TableName tableName : tablesInfo.keySet()) {
1922      if (isTableIncluded(tableName) && !tableStates.containsKey(tableName)) {
1923        if (fixMeta) {
1924          MetaTableAccessor.updateTableState(connection, tableName, TableState.State.ENABLED);
1925          TableState newState = MetaTableAccessor.getTableState(connection, tableName);
1926          if (newState == null) {
1927            errors.reportError(ERROR_CODE.NO_TABLE_STATE,
1928                "Unable to change state for table " + tableName + " in meta ");
1929          }
1930        } else {
1931          errors.reportError(ERROR_CODE.NO_TABLE_STATE,
1932              tableName + " has no state in meta ");
1933        }
1934      }
1935    }
1936  }
1937
1938  private void preCheckPermission() throws IOException, AccessDeniedException {
1939    if (shouldIgnorePreCheckPermission()) {
1940      return;
1941    }
1942
1943    Path hbaseDir = FSUtils.getRootDir(getConf());
1944    FileSystem fs = hbaseDir.getFileSystem(getConf());
1945    UserProvider userProvider = UserProvider.instantiate(getConf());
1946    UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1947    FileStatus[] files = fs.listStatus(hbaseDir);
1948    for (FileStatus file : files) {
1949      try {
1950        FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1951      } catch (AccessDeniedException ace) {
1952        LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1953        errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1954          + " does not have write perms to " + file.getPath()
1955          + ". Please rerun hbck as hdfs user " + file.getOwner());
1956        throw ace;
1957      }
1958    }
1959  }
1960
1961  /**
1962   * Deletes region from meta table
1963   */
1964  private void deleteMetaRegion(HbckRegionInfo hi) throws IOException {
1965    deleteMetaRegion(hi.getMetaEntry().getRegionName());
1966  }
1967
1968  /**
1969   * Deletes region from meta table
1970   */
1971  private void deleteMetaRegion(byte[] metaKey) throws IOException {
1972    Delete d = new Delete(metaKey);
1973    meta.delete(d);
1974    LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1975  }
1976
1977  /**
1978   * Reset the split parent region info in meta table
1979   */
1980  private void resetSplitParent(HbckRegionInfo hi) throws IOException {
1981    RowMutations mutations = new RowMutations(hi.getMetaEntry().getRegionName());
1982    Delete d = new Delete(hi.getMetaEntry().getRegionName());
1983    d.addColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1984    d.addColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1985    mutations.add(d);
1986
1987    RegionInfo hri = RegionInfoBuilder.newBuilder(hi.getMetaEntry())
1988        .setOffline(false)
1989        .setSplit(false)
1990        .build();
1991    Put p = MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
1992    mutations.add(p);
1993
1994    meta.mutateRow(mutations);
1995    LOG.info("Reset split parent " + hi.getMetaEntry().getRegionNameAsString() + " in META");
1996  }
1997
1998  /**
1999   * This backwards-compatibility wrapper for permanently offlining a region
2000   * that should not be alive.  If the region server does not support the
2001   * "offline" method, it will use the closest unassign method instead.  This
2002   * will basically work until one attempts to disable or delete the affected
2003   * table.  The problem has to do with in-memory only master state, so
2004   * restarting the HMaster or failing over to another should fix this.
2005   */
2006  void offline(byte[] regionName) throws IOException {
2007    String regionString = Bytes.toStringBinary(regionName);
2008    if (!rsSupportsOffline) {
2009      LOG.warn(
2010          "Using unassign region " + regionString + " instead of using offline method, you should" +
2011              " restart HMaster after these repairs");
2012      admin.unassign(regionName, true);
2013      return;
2014    }
2015
2016    // first time we assume the rs's supports #offline.
2017    try {
2018      LOG.info("Offlining region " + regionString);
2019      admin.offline(regionName);
2020    } catch (IOException ioe) {
2021      String notFoundMsg = "java.lang.NoSuchMethodException: " +
2022          "org.apache.hadoop.hbase.master.HMaster.offline([B)";
2023      if (ioe.getMessage().contains(notFoundMsg)) {
2024        LOG.warn("Using unassign region " + regionString +
2025            " instead of using offline method, you should" +
2026            " restart HMaster after these repairs");
2027        rsSupportsOffline = false; // in the future just use unassign
2028        admin.unassign(regionName, true);
2029        return;
2030      }
2031      throw ioe;
2032    }
2033  }
2034
2035  /**
2036   * Attempts to undeploy a region from a region server based in information in
2037   * META.  Any operations that modify the file system should make sure that
2038   * its corresponding region is not deployed to prevent data races.
2039   *
2040   * A separate call is required to update the master in-memory region state
2041   * kept in the AssignementManager.  Because disable uses this state instead of
2042   * that found in META, we can't seem to cleanly disable/delete tables that
2043   * have been hbck fixed.  When used on a version of HBase that does not have
2044   * the offline ipc call exposed on the master (&lt;0.90.5, &lt;0.92.0) a master
2045   * restart or failover may be required.
2046   */
2047  void closeRegion(HbckRegionInfo hi) throws IOException, InterruptedException {
2048    if (hi.getMetaEntry() == null && hi.getHdfsEntry() == null) {
2049      undeployRegions(hi);
2050      return;
2051    }
2052
2053    // get assignment info and hregioninfo from meta.
2054    Get get = new Get(hi.getRegionName());
2055    get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2056    get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
2057    get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
2058    // also get the locations of the replicas to close if the primary region is being closed
2059    if (hi.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
2060      int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2061      for (int i = 0; i < numReplicas; i++) {
2062        get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
2063        get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
2064      }
2065    }
2066    Result r = meta.get(get);
2067    RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
2068    if (rl == null) {
2069      LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
2070          " since meta does not have handle to reach it");
2071      return;
2072    }
2073    for (HRegionLocation h : rl.getRegionLocations()) {
2074      ServerName serverName = h.getServerName();
2075      if (serverName == null) {
2076        errors.reportError("Unable to close region "
2077            + hi.getRegionNameAsString() +  " because meta does not "
2078            + "have handle to reach it.");
2079        continue;
2080      }
2081      RegionInfo hri = h.getRegionInfo();
2082      if (hri == null) {
2083        LOG.warn("Unable to close region " + hi.getRegionNameAsString()
2084            + " because hbase:meta had invalid or missing "
2085            + HConstants.CATALOG_FAMILY_STR + ":"
2086            + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
2087            + " qualifier value.");
2088        continue;
2089      }
2090      // close the region -- close files and remove assignment
2091      HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
2092    }
2093  }
2094
2095  private void undeployRegions(HbckRegionInfo hi) throws IOException, InterruptedException {
2096    undeployRegionsForHbi(hi);
2097    // undeploy replicas of the region (but only if the method is invoked for the primary)
2098    if (hi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
2099      return;
2100    }
2101    int numReplicas = admin.getDescriptor(hi.getTableName()).getRegionReplication();
2102    for (int i = 1; i < numReplicas; i++) {
2103      if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
2104      RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
2105          hi.getPrimaryHRIForDeployedReplica(), i);
2106      HbckRegionInfo h = regionInfoMap.get(hri.getEncodedName());
2107      if (h != null) {
2108        undeployRegionsForHbi(h);
2109        //set skip checks; we undeployed it, and we don't want to evaluate this anymore
2110        //in consistency checks
2111        h.setSkipChecks(true);
2112      }
2113    }
2114  }
2115
2116  private void undeployRegionsForHbi(HbckRegionInfo hi) throws IOException, InterruptedException {
2117    for (HbckRegionInfo.OnlineEntry rse : hi.getOnlineEntries()) {
2118      LOG.debug("Undeploy region "  + rse.getRegionInfo() + " from " + rse.getServerName());
2119      try {
2120        HBaseFsckRepair
2121            .closeRegionSilentlyAndWait(connection, rse.getServerName(), rse.getRegionInfo());
2122        offline(rse.getRegionInfo().getRegionName());
2123      } catch (IOException ioe) {
2124        LOG.warn("Got exception when attempting to offline region "
2125            + Bytes.toString(rse.getRegionInfo().getRegionName()), ioe);
2126      }
2127    }
2128  }
2129
2130  private void tryAssignmentRepair(HbckRegionInfo hbi, String msg) throws IOException,
2131    KeeperException, InterruptedException {
2132    // If we are trying to fix the errors
2133    if (shouldFixAssignments()) {
2134      errors.print(msg);
2135      undeployRegions(hbi);
2136      setShouldRerun();
2137      RegionInfo hri = hbi.getHdfsHRI();
2138      if (hri == null) {
2139        hri = hbi.getMetaEntry();
2140      }
2141      HBaseFsckRepair.fixUnassigned(admin, hri);
2142      HBaseFsckRepair.waitUntilAssigned(admin, hri);
2143
2144      // also assign replicas if needed (do it only when this call operates on a primary replica)
2145      if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) return;
2146      int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
2147      for (int i = 1; i < replicationCount; i++) {
2148        hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
2149        HbckRegionInfo h = regionInfoMap.get(hri.getEncodedName());
2150        if (h != null) {
2151          undeployRegions(h);
2152          //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore
2153          //in consistency checks
2154          h.setSkipChecks(true);
2155        }
2156        HBaseFsckRepair.fixUnassigned(admin, hri);
2157        HBaseFsckRepair.waitUntilAssigned(admin, hri);
2158      }
2159
2160    }
2161  }
2162
2163  /**
2164   * Check a single region for consistency and correct deployment.
2165   */
2166  private void checkRegionConsistency(final String key, final HbckRegionInfo hbi)
2167      throws IOException, KeeperException, InterruptedException {
2168
2169    if (hbi.isSkipChecks()) return;
2170    String descriptiveName = hbi.toString();
2171    boolean inMeta = hbi.getMetaEntry() != null;
2172    // In case not checking HDFS, assume the region is on HDFS
2173    boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
2174    boolean hasMetaAssignment = inMeta && hbi.getMetaEntry().regionServer != null;
2175    boolean isDeployed = !hbi.getDeployedOn().isEmpty();
2176    boolean isMultiplyDeployed = hbi.getDeployedOn().size() > 1;
2177    boolean deploymentMatchesMeta =
2178      hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
2179      hbi.getMetaEntry().regionServer.equals(hbi.getDeployedOn().get(0));
2180    boolean splitParent =
2181        inMeta && hbi.getMetaEntry().isSplit() && hbi.getMetaEntry().isOffline();
2182    boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.getMetaEntry().getTable());
2183    boolean recentlyModified = inHdfs &&
2184      hbi.getModTime() + timelag > EnvironmentEdgeManager.currentTime();
2185
2186    // ========== First the healthy cases =============
2187    if (hbi.containsOnlyHdfsEdits()) {
2188      return;
2189    }
2190    if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
2191      return;
2192    } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
2193      LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
2194        "tabled that is not deployed");
2195      return;
2196    } else if (recentlyModified) {
2197      LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
2198      return;
2199    }
2200    // ========== Cases where the region is not in hbase:meta =============
2201    else if (!inMeta && !inHdfs && !isDeployed) {
2202      // We shouldn't have record of this region at all then!
2203      assert false : "Entry for region with no data";
2204    } else if (!inMeta && !inHdfs && isDeployed) {
2205      errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
2206          + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
2207          "deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));
2208      if (shouldFixAssignments()) {
2209        undeployRegions(hbi);
2210      }
2211
2212    } else if (!inMeta && inHdfs && !isDeployed) {
2213      if (hbi.isMerged()) {
2214        // This region has already been merged, the remaining hdfs file will be
2215        // cleaned by CatalogJanitor later
2216        hbi.setSkipChecks(true);
2217        LOG.info("Region " + descriptiveName
2218            + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
2219        return;
2220      }
2221      errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
2222          + descriptiveName + " on HDFS, but not listed in hbase:meta " +
2223          "or deployed on any region server");
2224      // restore region consistency of an adopted orphan
2225      if (shouldFixMeta()) {
2226        if (!hbi.isHdfsRegioninfoPresent()) {
2227          LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
2228              +  " in table integrity repair phase if -fixHdfsOrphans was" +
2229              " used.");
2230          return;
2231        }
2232
2233        RegionInfo hri = hbi.getHdfsHRI();
2234        HbckTableInfo tableInfo = tablesInfo.get(hri.getTable());
2235
2236        for (RegionInfo region : tableInfo.getRegionsFromMeta(this.regionInfoMap)) {
2237          if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0
2238              && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(),
2239                hri.getEndKey()) >= 0)
2240              && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) {
2241            if(region.isSplit() || region.isOffline()) continue;
2242            Path regionDir = hbi.getHdfsRegionDir();
2243            FileSystem fs = regionDir.getFileSystem(getConf());
2244            List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir);
2245            for (Path familyDir : familyDirs) {
2246              List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);
2247              for (Path referenceFilePath : referenceFilePaths) {
2248                Path parentRegionDir =
2249                    StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();
2250                if (parentRegionDir.toString().endsWith(region.getEncodedName())) {
2251                  LOG.warn(hri + " start and stop keys are in the range of " + region
2252                      + ". The region might not be cleaned up from hdfs when region " + region
2253                      + " split failed. Hence deleting from hdfs.");
2254                  HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
2255                    regionDir.getParent(), hri);
2256                  return;
2257                }
2258              }
2259            }
2260          }
2261        }
2262        LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
2263        int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2264        HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2265            admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
2266              .getLiveServerMetrics().keySet(), numReplicas);
2267
2268        tryAssignmentRepair(hbi, "Trying to reassign region...");
2269      }
2270
2271    } else if (!inMeta && inHdfs && isDeployed) {
2272      errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
2273          + " not in META, but deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));
2274      debugLsr(hbi.getHdfsRegionDir());
2275      if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
2276        // for replicas, this means that we should undeploy the region (we would have
2277        // gone over the primaries and fixed meta holes in first phase under
2278        // checkAndFixConsistency; we shouldn't get the condition !inMeta at
2279        // this stage unless unwanted replica)
2280        if (shouldFixAssignments()) {
2281          undeployRegionsForHbi(hbi);
2282        }
2283      }
2284      if (shouldFixMeta() && hbi.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
2285        if (!hbi.isHdfsRegioninfoPresent()) {
2286          LOG.error("This should have been repaired in table integrity repair phase");
2287          return;
2288        }
2289
2290        LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
2291        int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2292        HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2293            admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
2294              .getLiveServerMetrics().keySet(), numReplicas);
2295        tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2296      }
2297
2298    // ========== Cases where the region is in hbase:meta =============
2299    } else if (inMeta && inHdfs && !isDeployed && splitParent) {
2300      // check whether this is an actual error, or just transient state where parent
2301      // is not cleaned
2302      if (hbi.getMetaEntry().splitA != null && hbi.getMetaEntry().splitB != null) {
2303        // check that split daughters are there
2304        HbckRegionInfo infoA = this.regionInfoMap.get(hbi.getMetaEntry().splitA.getEncodedName());
2305        HbckRegionInfo infoB = this.regionInfoMap.get(hbi.getMetaEntry().splitB.getEncodedName());
2306        if (infoA != null && infoB != null) {
2307          // we already processed or will process daughters. Move on, nothing to see here.
2308          hbi.setSkipChecks(true);
2309          return;
2310        }
2311      }
2312
2313      // For Replica region, we need to do a similar check. If replica is not split successfully,
2314      // error is going to be reported against primary daughter region.
2315      if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
2316        LOG.info("Region " + descriptiveName + " is a split parent in META, in HDFS, "
2317            + "and not deployed on any region server. This may be transient.");
2318        hbi.setSkipChecks(true);
2319        return;
2320      }
2321
2322      errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2323          + descriptiveName + " is a split parent in META, in HDFS, "
2324          + "and not deployed on any region server. This could be transient, "
2325          + "consider to run the catalog janitor first!");
2326      if (shouldFixSplitParents()) {
2327        setShouldRerun();
2328        resetSplitParent(hbi);
2329      }
2330    } else if (inMeta && !inHdfs && !isDeployed) {
2331      errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2332          + descriptiveName + " found in META, but not in HDFS "
2333          + "or deployed on any region server.");
2334      if (shouldFixMeta()) {
2335        deleteMetaRegion(hbi);
2336      }
2337    } else if (inMeta && !inHdfs && isDeployed) {
2338      errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2339          + " found in META, but not in HDFS, " +
2340          "and deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));
2341      // We treat HDFS as ground truth.  Any information in meta is transient
2342      // and equivalent data can be regenerated.  So, lets unassign and remove
2343      // these problems from META.
2344      if (shouldFixAssignments()) {
2345        errors.print("Trying to fix unassigned region...");
2346        undeployRegions(hbi);
2347      }
2348      if (shouldFixMeta()) {
2349        // wait for it to complete
2350        deleteMetaRegion(hbi);
2351      }
2352    } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2353      errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2354          + " not deployed on any region server.");
2355      tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2356    } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2357      errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2358          "Region " + descriptiveName + " should not be deployed according " +
2359          "to META, but is deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));
2360      if (shouldFixAssignments()) {
2361        errors.print("Trying to close the region " + descriptiveName);
2362        setShouldRerun();
2363        HBaseFsckRepair.fixMultiAssignment(connection, hbi.getMetaEntry(), hbi.getDeployedOn());
2364      }
2365    } else if (inMeta && inHdfs && isMultiplyDeployed) {
2366      errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2367          + " is listed in hbase:meta on region server " + hbi.getMetaEntry().regionServer
2368          + " but is multiply assigned to region servers " +
2369          Joiner.on(", ").join(hbi.getDeployedOn()));
2370      // If we are trying to fix the errors
2371      if (shouldFixAssignments()) {
2372        errors.print("Trying to fix assignment error...");
2373        setShouldRerun();
2374        HBaseFsckRepair.fixMultiAssignment(connection, hbi.getMetaEntry(), hbi.getDeployedOn());
2375      }
2376    } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2377      errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2378          + descriptiveName + " listed in hbase:meta on region server " +
2379          hbi.getMetaEntry().regionServer + " but found on region server " +
2380          hbi.getDeployedOn().get(0));
2381      // If we are trying to fix the errors
2382      if (shouldFixAssignments()) {
2383        errors.print("Trying to fix assignment error...");
2384        setShouldRerun();
2385        HBaseFsckRepair.fixMultiAssignment(connection, hbi.getMetaEntry(), hbi.getDeployedOn());
2386        HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2387      }
2388    } else {
2389      errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2390          " is in an unforeseen state:" +
2391          " inMeta=" + inMeta +
2392          " inHdfs=" + inHdfs +
2393          " isDeployed=" + isDeployed +
2394          " isMultiplyDeployed=" + isMultiplyDeployed +
2395          " deploymentMatchesMeta=" + deploymentMatchesMeta +
2396          " shouldBeDeployed=" + shouldBeDeployed);
2397    }
2398  }
2399
2400  /**
2401   * Checks tables integrity. Goes over all regions and scans the tables.
2402   * Collects all the pieces for each table and checks if there are missing,
2403   * repeated or overlapping ones.
2404   * @throws IOException
2405   */
2406  SortedMap<TableName, HbckTableInfo> checkIntegrity() throws IOException {
2407    tablesInfo = new TreeMap<>();
2408    LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2409    for (HbckRegionInfo hbi : regionInfoMap.values()) {
2410      // Check only valid, working regions
2411      if (hbi.getMetaEntry() == null) {
2412        // this assumes that consistency check has run loadMetaEntry
2413        Path p = hbi.getHdfsRegionDir();
2414        if (p == null) {
2415          errors.report("No regioninfo in Meta or HDFS. " + hbi);
2416        }
2417
2418        // TODO test.
2419        continue;
2420      }
2421      if (hbi.getMetaEntry().regionServer == null) {
2422        errors.detail("Skipping region because no region server: " + hbi);
2423        continue;
2424      }
2425      if (hbi.getMetaEntry().isOffline()) {
2426        errors.detail("Skipping region because it is offline: " + hbi);
2427        continue;
2428      }
2429      if (hbi.containsOnlyHdfsEdits()) {
2430        errors.detail("Skipping region because it only contains edits" + hbi);
2431        continue;
2432      }
2433
2434      // Missing regionDir or over-deployment is checked elsewhere. Include
2435      // these cases in modTInfo, so we can evaluate those regions as part of
2436      // the region chain in META
2437      //if (hbi.foundRegionDir == null) continue;
2438      //if (hbi.deployedOn.size() != 1) continue;
2439      if (hbi.getDeployedOn().isEmpty()) {
2440        continue;
2441      }
2442
2443      // We should be safe here
2444      TableName tableName = hbi.getMetaEntry().getTable();
2445      HbckTableInfo modTInfo = tablesInfo.get(tableName);
2446      if (modTInfo == null) {
2447        modTInfo = new HbckTableInfo(tableName, this);
2448      }
2449      for (ServerName server : hbi.getDeployedOn()) {
2450        modTInfo.addServer(server);
2451      }
2452
2453      if (!hbi.isSkipChecks()) {
2454        modTInfo.addRegionInfo(hbi);
2455      }
2456
2457      tablesInfo.put(tableName, modTInfo);
2458    }
2459
2460    loadTableInfosForTablesWithNoRegion();
2461
2462    logParallelMerge();
2463    for (HbckTableInfo tInfo : tablesInfo.values()) {
2464      TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2465      if (!tInfo.checkRegionChain(handler)) {
2466        errors.report("Found inconsistency in table " + tInfo.getName());
2467      }
2468    }
2469    return tablesInfo;
2470  }
2471
2472  /** Loads table info's for tables that may not have been included, since there are no
2473   * regions reported for the table, but table dir is there in hdfs
2474   */
2475  private void loadTableInfosForTablesWithNoRegion() throws IOException {
2476    Map<String, TableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2477    for (TableDescriptor htd : allTables.values()) {
2478      if (checkMetaOnly && !htd.isMetaTable()) {
2479        continue;
2480      }
2481
2482      TableName tableName = htd.getTableName();
2483      if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2484        HbckTableInfo tableInfo = new HbckTableInfo(tableName, this);
2485        tableInfo.htds.add(htd);
2486        tablesInfo.put(htd.getTableName(), tableInfo);
2487      }
2488    }
2489  }
2490
2491  /**
2492   * Merge hdfs data by moving from contained HbckRegionInfo into targetRegionDir.
2493   * @return number of file move fixes done to merge regions.
2494   */
2495  public int mergeRegionDirs(Path targetRegionDir, HbckRegionInfo contained) throws IOException {
2496    int fileMoves = 0;
2497    String thread = Thread.currentThread().getName();
2498    LOG.debug("[" + thread + "] Contained region dir after close and pause");
2499    debugLsr(contained.getHdfsRegionDir());
2500
2501    // rename the contained into the container.
2502    FileSystem fs = targetRegionDir.getFileSystem(getConf());
2503    FileStatus[] dirs = null;
2504    try {
2505      dirs = fs.listStatus(contained.getHdfsRegionDir());
2506    } catch (FileNotFoundException fnfe) {
2507      // region we are attempting to merge in is not present!  Since this is a merge, there is
2508      // no harm skipping this region if it does not exist.
2509      if (!fs.exists(contained.getHdfsRegionDir())) {
2510        LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2511            + " is missing. Assuming already sidelined or moved.");
2512      } else {
2513        sidelineRegionDir(fs, contained);
2514      }
2515      return fileMoves;
2516    }
2517
2518    if (dirs == null) {
2519      if (!fs.exists(contained.getHdfsRegionDir())) {
2520        LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2521            + " already sidelined.");
2522      } else {
2523        sidelineRegionDir(fs, contained);
2524      }
2525      return fileMoves;
2526    }
2527
2528    for (FileStatus cf : dirs) {
2529      Path src = cf.getPath();
2530      Path dst =  new Path(targetRegionDir, src.getName());
2531
2532      if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2533        // do not copy the old .regioninfo file.
2534        continue;
2535      }
2536
2537      if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2538        // do not copy the .oldlogs files
2539        continue;
2540      }
2541
2542      LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2543      // FileSystem.rename is inconsistent with directories -- if the
2544      // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2545      // it moves the src into the dst dir resulting in (foo/a/b).  If
2546      // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2547      for (FileStatus hfile : fs.listStatus(src)) {
2548        boolean success = fs.rename(hfile.getPath(), dst);
2549        if (success) {
2550          fileMoves++;
2551        }
2552      }
2553      LOG.debug("[" + thread + "] Sideline directory contents:");
2554      debugLsr(targetRegionDir);
2555    }
2556
2557    // if all success.
2558    sidelineRegionDir(fs, contained);
2559    LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2560        getSidelineDir());
2561    debugLsr(contained.getHdfsRegionDir());
2562
2563    return fileMoves;
2564  }
2565
2566
2567  static class WorkItemOverlapMerge implements Callable<Void> {
2568    private TableIntegrityErrorHandler handler;
2569    Collection<HbckRegionInfo> overlapgroup;
2570
2571    WorkItemOverlapMerge(Collection<HbckRegionInfo> overlapgroup,
2572        TableIntegrityErrorHandler handler) {
2573      this.handler = handler;
2574      this.overlapgroup = overlapgroup;
2575    }
2576
2577    @Override
2578    public Void call() throws Exception {
2579      handler.handleOverlapGroup(overlapgroup);
2580      return null;
2581    }
2582  };
2583
2584  /**
2585   * Return a list of user-space table names whose metadata have not been
2586   * modified in the last few milliseconds specified by timelag
2587   * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
2588   * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
2589   * milliseconds specified by timelag, then the table is a candidate to be returned.
2590   * @return tables that have not been modified recently
2591   * @throws IOException if an error is encountered
2592   */
2593  TableDescriptor[] getTables(AtomicInteger numSkipped) {
2594    List<TableName> tableNames = new ArrayList<>();
2595    long now = EnvironmentEdgeManager.currentTime();
2596
2597    for (HbckRegionInfo hbi : regionInfoMap.values()) {
2598      HbckRegionInfo.MetaEntry info = hbi.getMetaEntry();
2599
2600      // if the start key is zero, then we have found the first region of a table.
2601      // pick only those tables that were not modified in the last few milliseconds.
2602      if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
2603        if (info.modTime + timelag < now) {
2604          tableNames.add(info.getTable());
2605        } else {
2606          numSkipped.incrementAndGet(); // one more in-flux table
2607        }
2608      }
2609    }
2610    return getTableDescriptors(tableNames);
2611  }
2612
2613  TableDescriptor[] getTableDescriptors(List<TableName> tableNames) {
2614      LOG.info("getTableDescriptors == tableNames => " + tableNames);
2615    try (Connection conn = ConnectionFactory.createConnection(getConf());
2616        Admin admin = conn.getAdmin()) {
2617      List<TableDescriptor> tds = admin.listTableDescriptors(tableNames);
2618      return tds.toArray(new TableDescriptor[tds.size()]);
2619    } catch (IOException e) {
2620      LOG.debug("Exception getting table descriptors", e);
2621    }
2622    return new TableDescriptor[0];
2623  }
2624
2625  /**
2626   * Gets the entry in regionInfo corresponding to the the given encoded
2627   * region name. If the region has not been seen yet, a new entry is added
2628   * and returned.
2629   */
2630  private synchronized HbckRegionInfo getOrCreateInfo(String name) {
2631    HbckRegionInfo hbi = regionInfoMap.get(name);
2632    if (hbi == null) {
2633      hbi = new HbckRegionInfo(null);
2634      regionInfoMap.put(name, hbi);
2635    }
2636    return hbi;
2637  }
2638
2639  private void checkAndFixReplication() throws ReplicationException {
2640    ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, errors);
2641    checker.checkUnDeletedQueues();
2642
2643    if (checker.hasUnDeletedQueues() && this.fixReplication) {
2644      checker.fixUnDeletedQueues();
2645      setShouldRerun();
2646    }
2647  }
2648
2649  /**
2650    * Check values in regionInfo for hbase:meta
2651    * Check if zero or more than one regions with hbase:meta are found.
2652    * If there are inconsistencies (i.e. zero or more than one regions
2653    * pretend to be holding the hbase:meta) try to fix that and report an error.
2654    * @throws IOException from HBaseFsckRepair functions
2655    * @throws KeeperException
2656    * @throws InterruptedException
2657    */
2658  boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
2659    Map<Integer, HbckRegionInfo> metaRegions = new HashMap<>();
2660    for (HbckRegionInfo value : regionInfoMap.values()) {
2661      if (value.getMetaEntry() != null && value.getMetaEntry().isMetaRegion()) {
2662        metaRegions.put(value.getReplicaId(), value);
2663      }
2664    }
2665    int metaReplication = admin.getTableDescriptor(TableName.META_TABLE_NAME)
2666        .getRegionReplication();
2667    boolean noProblem = true;
2668    // There will be always entries in regionInfoMap corresponding to hbase:meta & its replicas
2669    // Check the deployed servers. It should be exactly one server for each replica.
2670    for (int i = 0; i < metaReplication; i++) {
2671      HbckRegionInfo metaHbckRegionInfo = metaRegions.remove(i);
2672      List<ServerName> servers = new ArrayList<>();
2673      if (metaHbckRegionInfo != null) {
2674        servers = metaHbckRegionInfo.getDeployedOn();
2675      }
2676      if (servers.size() != 1) {
2677        noProblem = false;
2678        if (servers.isEmpty()) {
2679          assignMetaReplica(i);
2680        } else if (servers.size() > 1) {
2681          errors
2682          .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId " +
2683                       metaHbckRegionInfo.getReplicaId() + " is found on more than one region.");
2684          if (shouldFixAssignments()) {
2685            errors.print("Trying to fix a problem with hbase:meta, replicaId " +
2686                metaHbckRegionInfo.getReplicaId() + "..");
2687            setShouldRerun();
2688            // try fix it (treat is a dupe assignment)
2689            HBaseFsckRepair
2690                .fixMultiAssignment(connection, metaHbckRegionInfo.getMetaEntry(), servers);
2691          }
2692        }
2693      }
2694    }
2695    // unassign whatever is remaining in metaRegions. They are excess replicas.
2696    for (Map.Entry<Integer, HbckRegionInfo> entry : metaRegions.entrySet()) {
2697      noProblem = false;
2698      errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2699          "hbase:meta replicas are deployed in excess. Configured " + metaReplication +
2700          ", deployed " + metaRegions.size());
2701      if (shouldFixAssignments()) {
2702        errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() +
2703            " of hbase:meta..");
2704        setShouldRerun();
2705        unassignMetaReplica(entry.getValue());
2706      }
2707    }
2708    // if noProblem is false, rerun hbck with hopefully fixed META
2709    // if noProblem is true, no errors, so continue normally
2710    return noProblem;
2711  }
2712
2713  private void unassignMetaReplica(HbckRegionInfo hi)
2714      throws IOException, InterruptedException, KeeperException {
2715    undeployRegions(hi);
2716    ZKUtil
2717        .deleteNode(zkw, zkw.getZNodePaths().getZNodeForReplica(hi.getMetaEntry().getReplicaId()));
2718  }
2719
2720  private void assignMetaReplica(int replicaId)
2721      throws IOException, KeeperException, InterruptedException {
2722    errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " +
2723        replicaId +" is not found on any region.");
2724    if (shouldFixAssignments()) {
2725      errors.print("Trying to fix a problem with hbase:meta..");
2726      setShouldRerun();
2727      // try to fix it (treat it as unassigned region)
2728      RegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
2729          RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId);
2730      HBaseFsckRepair.fixUnassigned(admin, h);
2731      HBaseFsckRepair.waitUntilAssigned(admin, h);
2732    }
2733  }
2734
2735  /**
2736   * Scan hbase:meta, adding all regions found to the regionInfo map.
2737   * @throws IOException if an error is encountered
2738   */
2739  boolean loadMetaEntries() throws IOException {
2740    MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
2741      int countRecord = 1;
2742
2743      // comparator to sort KeyValues with latest modtime
2744      final Comparator<Cell> comp = new Comparator<Cell>() {
2745        @Override
2746        public int compare(Cell k1, Cell k2) {
2747          return Long.compare(k1.getTimestamp(), k2.getTimestamp());
2748        }
2749      };
2750
2751      @Override
2752      public boolean visit(Result result) throws IOException {
2753        try {
2754
2755          // record the latest modification of this META record
2756          long ts =  Collections.max(result.listCells(), comp).getTimestamp();
2757          RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
2758          if (rl == null) {
2759            emptyRegionInfoQualifiers.add(result);
2760            errors.reportError(ERROR_CODE.EMPTY_META_CELL,
2761              "Empty REGIONINFO_QUALIFIER found in hbase:meta");
2762            return true;
2763          }
2764          ServerName sn = null;
2765          if (rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID) == null ||
2766              rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
2767            emptyRegionInfoQualifiers.add(result);
2768            errors.reportError(ERROR_CODE.EMPTY_META_CELL,
2769              "Empty REGIONINFO_QUALIFIER found in hbase:meta");
2770            return true;
2771          }
2772          RegionInfo hri = rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
2773          if (!(isTableIncluded(hri.getTable())
2774              || hri.isMetaRegion())) {
2775            return true;
2776          }
2777          PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(result);
2778          for (HRegionLocation h : rl.getRegionLocations()) {
2779            if (h == null || h.getRegionInfo() == null) {
2780              continue;
2781            }
2782            sn = h.getServerName();
2783            hri = h.getRegionInfo();
2784
2785            HbckRegionInfo.MetaEntry m = null;
2786            if (hri.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
2787              m = new HbckRegionInfo.MetaEntry(hri, sn, ts, daughters.getFirst(),
2788                  daughters.getSecond());
2789            } else {
2790              m = new HbckRegionInfo.MetaEntry(hri, sn, ts, null, null);
2791            }
2792            HbckRegionInfo previous = regionInfoMap.get(hri.getEncodedName());
2793            if (previous == null) {
2794              regionInfoMap.put(hri.getEncodedName(), new HbckRegionInfo(m));
2795            } else if (previous.getMetaEntry() == null) {
2796              previous.setMetaEntry(m);
2797            } else {
2798              throw new IOException("Two entries in hbase:meta are same " + previous);
2799            }
2800          }
2801          List<RegionInfo> mergeParents = MetaTableAccessor.getMergeRegions(result.rawCells());
2802          if (mergeParents != null) {
2803            for (RegionInfo mergeRegion : mergeParents) {
2804              if (mergeRegion != null) {
2805                // This region is already being merged
2806                HbckRegionInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
2807                hbInfo.setMerged(true);
2808              }
2809            }
2810          }
2811
2812          // show proof of progress to the user, once for every 100 records.
2813          if (countRecord % 100 == 0) {
2814            errors.progress();
2815          }
2816          countRecord++;
2817          return true;
2818        } catch (RuntimeException e) {
2819          LOG.error("Result=" + result);
2820          throw e;
2821        }
2822      }
2823    };
2824    if (!checkMetaOnly) {
2825      // Scan hbase:meta to pick up user regions
2826      MetaTableAccessor.fullScanRegions(connection, visitor);
2827    }
2828
2829    errors.print("");
2830    return true;
2831  }
2832
2833  /**
2834   * Prints summary of all tables found on the system.
2835   */
2836  private void printTableSummary(SortedMap<TableName, HbckTableInfo> tablesInfo) {
2837    StringBuilder sb = new StringBuilder();
2838    int numOfSkippedRegions;
2839    errors.print("Summary:");
2840    for (HbckTableInfo tInfo : tablesInfo.values()) {
2841      numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName())) ?
2842          skippedRegions.get(tInfo.getName()).size() : 0;
2843
2844      if (errors.tableHasErrors(tInfo)) {
2845        errors.print("Table " + tInfo.getName() + " is inconsistent.");
2846      } else if (numOfSkippedRegions > 0){
2847        errors.print("Table " + tInfo.getName() + " is okay (with "
2848          + numOfSkippedRegions + " skipped regions).");
2849      }
2850      else {
2851        errors.print("Table " + tInfo.getName() + " is okay.");
2852      }
2853      errors.print("    Number of regions: " + tInfo.getNumRegions());
2854      if (numOfSkippedRegions > 0) {
2855        Set<String> skippedRegionStrings = skippedRegions.get(tInfo.getName());
2856        System.out.println("    Number of skipped regions: " + numOfSkippedRegions);
2857        System.out.println("      List of skipped regions:");
2858        for(String sr : skippedRegionStrings) {
2859          System.out.println("        " + sr);
2860        }
2861      }
2862      sb.setLength(0); // clear out existing buffer, if any.
2863      sb.append("    Deployed on: ");
2864      for (ServerName server : tInfo.deployedOn) {
2865        sb.append(" " + server.toString());
2866      }
2867      errors.print(sb.toString());
2868    }
2869  }
2870
2871  static HbckErrorReporter getErrorReporter(final Configuration conf)
2872      throws ClassNotFoundException {
2873    Class<? extends HbckErrorReporter> reporter =
2874        conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class,
2875            HbckErrorReporter.class);
2876    return ReflectionUtils.newInstance(reporter, conf);
2877  }
2878
2879  static class PrintingErrorReporter implements HbckErrorReporter {
2880    public int errorCount = 0;
2881    private int showProgress;
2882    // How frequently calls to progress() will create output
2883    private static final int progressThreshold = 100;
2884
2885    Set<HbckTableInfo> errorTables = new HashSet<>();
2886
2887    // for use by unit tests to verify which errors were discovered
2888    private ArrayList<ERROR_CODE> errorList = new ArrayList<>();
2889
2890    @Override
2891    public void clear() {
2892      errorTables.clear();
2893      errorList.clear();
2894      errorCount = 0;
2895    }
2896
2897    @Override
2898    public synchronized void reportError(ERROR_CODE errorCode, String message) {
2899      if (errorCode == ERROR_CODE.WRONG_USAGE) {
2900        System.err.println(message);
2901        return;
2902      }
2903
2904      errorList.add(errorCode);
2905      if (!summary) {
2906        System.out.println("ERROR: " + message);
2907      }
2908      errorCount++;
2909      showProgress = 0;
2910    }
2911
2912    @Override
2913    public synchronized void reportError(ERROR_CODE errorCode, String message,
2914        HbckTableInfo table) {
2915      errorTables.add(table);
2916      reportError(errorCode, message);
2917    }
2918
2919    @Override
2920    public synchronized void reportError(ERROR_CODE errorCode, String message, HbckTableInfo table,
2921                                         HbckRegionInfo info) {
2922      errorTables.add(table);
2923      String reference = "(region " + info.getRegionNameAsString() + ")";
2924      reportError(errorCode, reference + " " + message);
2925    }
2926
2927    @Override
2928    public synchronized void reportError(ERROR_CODE errorCode, String message, HbckTableInfo table,
2929                                         HbckRegionInfo info1, HbckRegionInfo info2) {
2930      errorTables.add(table);
2931      String reference = "(regions " + info1.getRegionNameAsString()
2932          + " and " + info2.getRegionNameAsString() + ")";
2933      reportError(errorCode, reference + " " + message);
2934    }
2935
2936    @Override
2937    public synchronized void reportError(String message) {
2938      reportError(ERROR_CODE.UNKNOWN, message);
2939    }
2940
2941    /**
2942     * Report error information, but do not increment the error count.  Intended for cases
2943     * where the actual error would have been reported previously.
2944     * @param message
2945     */
2946    @Override
2947    public synchronized void report(String message) {
2948      if (! summary) {
2949        System.out.println("ERROR: " + message);
2950      }
2951      showProgress = 0;
2952    }
2953
2954    @Override
2955    public synchronized int summarize() {
2956      System.out.println(Integer.toString(errorCount) +
2957                         " inconsistencies detected.");
2958      if (errorCount == 0) {
2959        System.out.println("Status: OK");
2960        return 0;
2961      } else {
2962        System.out.println("Status: INCONSISTENT");
2963        return -1;
2964      }
2965    }
2966
2967    @Override
2968    public ArrayList<ERROR_CODE> getErrorList() {
2969      return errorList;
2970    }
2971
2972    @Override
2973    public synchronized void print(String message) {
2974      if (!summary) {
2975        System.out.println(message);
2976      }
2977    }
2978
2979    @Override
2980    public boolean tableHasErrors(HbckTableInfo table) {
2981      return errorTables.contains(table);
2982    }
2983
2984    @Override
2985    public void resetErrors() {
2986      errorCount = 0;
2987    }
2988
2989    @Override
2990    public synchronized void detail(String message) {
2991      if (details) {
2992        System.out.println(message);
2993      }
2994      showProgress = 0;
2995    }
2996
2997    @Override
2998    public synchronized void progress() {
2999      if (showProgress++ == progressThreshold) {
3000        if (!summary) {
3001          System.out.print(".");
3002        }
3003        showProgress = 0;
3004      }
3005    }
3006  }
3007
3008  /**
3009   * Contact a region server and get all information from it
3010   */
3011  static class WorkItemRegion implements Callable<Void> {
3012    private final HBaseFsck hbck;
3013    private final ServerName rsinfo;
3014    private final HbckErrorReporter errors;
3015    private final ClusterConnection connection;
3016
3017    WorkItemRegion(HBaseFsck hbck, ServerName info, HbckErrorReporter errors,
3018        ClusterConnection connection) {
3019      this.hbck = hbck;
3020      this.rsinfo = info;
3021      this.errors = errors;
3022      this.connection = connection;
3023    }
3024
3025    @Override
3026    public synchronized Void call() throws IOException {
3027      errors.progress();
3028      try {
3029        BlockingInterface server = connection.getAdmin(rsinfo);
3030
3031        // list all online regions from this region server
3032        List<RegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
3033        regions = filterRegions(regions);
3034
3035        if (details) {
3036          errors.detail("RegionServer: " + rsinfo.getServerName() +
3037                           " number of regions: " + regions.size());
3038          for (RegionInfo rinfo: regions) {
3039            errors.detail("  " + rinfo.getRegionNameAsString() +
3040                             " id: " + rinfo.getRegionId() +
3041                             " encoded_name: " + rinfo.getEncodedName() +
3042                             " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
3043                             " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
3044          }
3045        }
3046
3047        // check to see if the existence of this region matches the region in META
3048
3049        for (RegionInfo r : regions) {
3050          HbckRegionInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
3051          hbi.addServer(r, rsinfo);
3052        }
3053      } catch (IOException e) {          // unable to connect to the region server.
3054        errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
3055          " Unable to fetch region information. " + e);
3056        throw e;
3057      }
3058      return null;
3059    }
3060
3061    private List<RegionInfo> filterRegions(List<RegionInfo> regions) {
3062      List<RegionInfo> ret = Lists.newArrayList();
3063      for (RegionInfo hri : regions) {
3064        if (hri.isMetaRegion() || (!hbck.checkMetaOnly
3065            && hbck.isTableIncluded(hri.getTable()))) {
3066          ret.add(hri);
3067        }
3068      }
3069      return ret;
3070    }
3071  }
3072
3073  /**
3074   * Contact hdfs and get all information about specified table directory into
3075   * regioninfo list.
3076   */
3077  class WorkItemHdfsDir implements Callable<Void> {
3078    private FileStatus tableDir;
3079    private HbckErrorReporter errors;
3080    private FileSystem fs;
3081
3082    WorkItemHdfsDir(FileSystem fs, HbckErrorReporter errors, FileStatus status) {
3083      this.fs = fs;
3084      this.tableDir = status;
3085      this.errors = errors;
3086    }
3087
3088    @Override
3089    public synchronized Void call() throws InterruptedException, ExecutionException {
3090      final Vector<Exception> exceptions = new Vector<>();
3091
3092      try {
3093        final FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
3094        final List<Future<?>> futures = new ArrayList<>(regionDirs.length);
3095
3096        for (final FileStatus regionDir : regionDirs) {
3097          errors.progress();
3098          final String encodedName = regionDir.getPath().getName();
3099          // ignore directories that aren't hexadecimal
3100          if (!encodedName.toLowerCase(Locale.ROOT).matches("[0-9a-f]+")) {
3101            continue;
3102          }
3103
3104          if (!exceptions.isEmpty()) {
3105            break;
3106          }
3107
3108          futures.add(executor.submit(new Runnable() {
3109            @Override
3110            public void run() {
3111              try {
3112                LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
3113
3114                Path regioninfoFile = new Path(regionDir.getPath(), HRegionFileSystem.REGION_INFO_FILE);
3115                boolean regioninfoFileExists = fs.exists(regioninfoFile);
3116
3117                if (!regioninfoFileExists) {
3118                  // As tables become larger it is more and more likely that by the time you
3119                  // reach a given region that it will be gone due to region splits/merges.
3120                  if (!fs.exists(regionDir.getPath())) {
3121                    LOG.warn("By the time we tried to process this region dir it was already gone: "
3122                        + regionDir.getPath());
3123                    return;
3124                  }
3125                }
3126
3127                HbckRegionInfo hbi = HBaseFsck.this.getOrCreateInfo(encodedName);
3128                HbckRegionInfo.HdfsEntry he = new HbckRegionInfo.HdfsEntry();
3129                synchronized (hbi) {
3130                  if (hbi.getHdfsRegionDir() != null) {
3131                    errors.print("Directory " + encodedName + " duplicate??" +
3132                                 hbi.getHdfsRegionDir());
3133                  }
3134
3135                  he.regionDir = regionDir.getPath();
3136                  he.regionDirModTime = regionDir.getModificationTime();
3137                  he.hdfsRegioninfoFilePresent = regioninfoFileExists;
3138                  // we add to orphan list when we attempt to read .regioninfo
3139
3140                  // Set a flag if this region contains only edits
3141                  // This is special case if a region is left after split
3142                  he.hdfsOnlyEdits = true;
3143                  FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
3144                  Path ePath = WALSplitUtil.getRegionDirRecoveredEditsDir(regionDir.getPath());
3145                  for (FileStatus subDir : subDirs) {
3146                    errors.progress();
3147                    String sdName = subDir.getPath().getName();
3148                    if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
3149                      he.hdfsOnlyEdits = false;
3150                      break;
3151                    }
3152                  }
3153                  hbi.setHdfsEntry(he);
3154                }
3155              } catch (Exception e) {
3156                LOG.error("Could not load region dir", e);
3157                exceptions.add(e);
3158              }
3159            }
3160          }));
3161        }
3162
3163        // Ensure all pending tasks are complete (or that we run into an exception)
3164        for (Future<?> f : futures) {
3165          if (!exceptions.isEmpty()) {
3166            break;
3167          }
3168          try {
3169            f.get();
3170          } catch (ExecutionException e) {
3171            LOG.error("Unexpected exec exception!  Should've been caught already.  (Bug?)", e);
3172            // Shouldn't happen, we already logged/caught any exceptions in the Runnable
3173          };
3174        }
3175      } catch (IOException e) {
3176        LOG.error("Cannot execute WorkItemHdfsDir for " + tableDir, e);
3177        exceptions.add(e);
3178      } finally {
3179        if (!exceptions.isEmpty()) {
3180          errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
3181              + tableDir.getPath().getName()
3182              + " Unable to fetch all HDFS region information. ");
3183          // Just throw the first exception as an indication something bad happened
3184          // Don't need to propagate all the exceptions, we already logged them all anyway
3185          throw new ExecutionException("First exception in WorkItemHdfsDir", exceptions.firstElement());
3186        }
3187      }
3188      return null;
3189    }
3190  }
3191
3192  /**
3193   * Contact hdfs and get all information about specified table directory into
3194   * regioninfo list.
3195   */
3196  static class WorkItemHdfsRegionInfo implements Callable<Void> {
3197    private HbckRegionInfo hbi;
3198    private HBaseFsck hbck;
3199    private HbckErrorReporter errors;
3200
3201    WorkItemHdfsRegionInfo(HbckRegionInfo hbi, HBaseFsck hbck, HbckErrorReporter errors) {
3202      this.hbi = hbi;
3203      this.hbck = hbck;
3204      this.errors = errors;
3205    }
3206
3207    @Override
3208    public synchronized Void call() throws IOException {
3209      // only load entries that haven't been loaded yet.
3210      if (hbi.getHdfsHRI() == null) {
3211        try {
3212          errors.progress();
3213          hbi.loadHdfsRegioninfo(hbck.getConf());
3214        } catch (IOException ioe) {
3215          String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
3216              + hbi.getTableName() + " in hdfs dir "
3217              + hbi.getHdfsRegionDir()
3218              + "!  It may be an invalid format or version file.  Treating as "
3219              + "an orphaned regiondir.";
3220          errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
3221          try {
3222            hbck.debugLsr(hbi.getHdfsRegionDir());
3223          } catch (IOException ioe2) {
3224            LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
3225            throw ioe2;
3226          }
3227          hbck.orphanHdfsDirs.add(hbi);
3228          throw ioe;
3229        }
3230      }
3231      return null;
3232    }
3233  };
3234
3235  /**
3236   * Display the full report from fsck. This displays all live and dead region
3237   * servers, and all known regions.
3238   */
3239  public static void setDisplayFullReport() {
3240    details = true;
3241  }
3242
3243  public static boolean shouldDisplayFullReport() {
3244    return details;
3245  }
3246
3247  /**
3248   * Set exclusive mode.
3249   */
3250  public static void setForceExclusive() {
3251    forceExclusive = true;
3252  }
3253
3254  /**
3255   * Only one instance of hbck can modify HBase at a time.
3256   */
3257  public boolean isExclusive() {
3258    return fixAny || forceExclusive;
3259  }
3260
3261  /**
3262   * Set summary mode.
3263   * Print only summary of the tables and status (OK or INCONSISTENT)
3264   */
3265  static void setSummary() {
3266    summary = true;
3267  }
3268
3269  /**
3270   * Set hbase:meta check mode.
3271   * Print only info about hbase:meta table deployment/state
3272   */
3273  void setCheckMetaOnly() {
3274    checkMetaOnly = true;
3275  }
3276
3277  /**
3278   * Set region boundaries check mode.
3279   */
3280  void setRegionBoundariesCheck() {
3281    checkRegionBoundaries = true;
3282  }
3283
3284  /**
3285   * Set replication fix mode.
3286   */
3287  public void setFixReplication(boolean shouldFix) {
3288    fixReplication = shouldFix;
3289    fixAny |= shouldFix;
3290  }
3291
3292  public void setCleanReplicationBarrier(boolean shouldClean) {
3293    cleanReplicationBarrier = shouldClean;
3294  }
3295
3296  /**
3297   * Check if we should rerun fsck again. This checks if we've tried to
3298   * fix something and we should rerun fsck tool again.
3299   * Display the full report from fsck. This displays all live and dead
3300   * region servers, and all known regions.
3301   */
3302  void setShouldRerun() {
3303    rerun = true;
3304  }
3305
3306  public boolean shouldRerun() {
3307    return rerun;
3308  }
3309
3310  /**
3311   * Fix inconsistencies found by fsck. This should try to fix errors (if any)
3312   * found by fsck utility.
3313   */
3314  public void setFixAssignments(boolean shouldFix) {
3315    fixAssignments = shouldFix;
3316    fixAny |= shouldFix;
3317  }
3318
3319  boolean shouldFixAssignments() {
3320    return fixAssignments;
3321  }
3322
3323  public void setFixMeta(boolean shouldFix) {
3324    fixMeta = shouldFix;
3325    fixAny |= shouldFix;
3326  }
3327
3328  boolean shouldFixMeta() {
3329    return fixMeta;
3330  }
3331
3332  public void setFixEmptyMetaCells(boolean shouldFix) {
3333    fixEmptyMetaCells = shouldFix;
3334    fixAny |= shouldFix;
3335  }
3336
3337  boolean shouldFixEmptyMetaCells() {
3338    return fixEmptyMetaCells;
3339  }
3340
3341  public void setCheckHdfs(boolean checking) {
3342    checkHdfs = checking;
3343  }
3344
3345  boolean shouldCheckHdfs() {
3346    return checkHdfs;
3347  }
3348
3349  public void setFixHdfsHoles(boolean shouldFix) {
3350    fixHdfsHoles = shouldFix;
3351    fixAny |= shouldFix;
3352  }
3353
3354  boolean shouldFixHdfsHoles() {
3355    return fixHdfsHoles;
3356  }
3357
3358  public void setFixTableOrphans(boolean shouldFix) {
3359    fixTableOrphans = shouldFix;
3360    fixAny |= shouldFix;
3361  }
3362
3363  boolean shouldFixTableOrphans() {
3364    return fixTableOrphans;
3365  }
3366
3367  public void setFixHdfsOverlaps(boolean shouldFix) {
3368    fixHdfsOverlaps = shouldFix;
3369    fixAny |= shouldFix;
3370  }
3371
3372  boolean shouldFixHdfsOverlaps() {
3373    return fixHdfsOverlaps;
3374  }
3375
3376  public void setFixHdfsOrphans(boolean shouldFix) {
3377    fixHdfsOrphans = shouldFix;
3378    fixAny |= shouldFix;
3379  }
3380
3381  boolean shouldFixHdfsOrphans() {
3382    return fixHdfsOrphans;
3383  }
3384
3385  public void setFixVersionFile(boolean shouldFix) {
3386    fixVersionFile = shouldFix;
3387    fixAny |= shouldFix;
3388  }
3389
3390  public boolean shouldFixVersionFile() {
3391    return fixVersionFile;
3392  }
3393
3394  public void setSidelineBigOverlaps(boolean sbo) {
3395    this.sidelineBigOverlaps = sbo;
3396  }
3397
3398  public boolean shouldSidelineBigOverlaps() {
3399    return sidelineBigOverlaps;
3400  }
3401
3402  public void setFixSplitParents(boolean shouldFix) {
3403    fixSplitParents = shouldFix;
3404    fixAny |= shouldFix;
3405  }
3406
3407  public void setRemoveParents(boolean shouldFix) {
3408    removeParents = shouldFix;
3409    fixAny |= shouldFix;
3410  }
3411
3412  boolean shouldFixSplitParents() {
3413    return fixSplitParents;
3414  }
3415
3416  boolean shouldRemoveParents() {
3417    return removeParents;
3418  }
3419
3420  public void setFixReferenceFiles(boolean shouldFix) {
3421    fixReferenceFiles = shouldFix;
3422    fixAny |= shouldFix;
3423  }
3424
3425  boolean shouldFixReferenceFiles() {
3426    return fixReferenceFiles;
3427  }
3428
3429  public void setFixHFileLinks(boolean shouldFix) {
3430    fixHFileLinks = shouldFix;
3431    fixAny |= shouldFix;
3432  }
3433
3434  boolean shouldFixHFileLinks() {
3435    return fixHFileLinks;
3436  }
3437
3438  public boolean shouldIgnorePreCheckPermission() {
3439    return !fixAny || ignorePreCheckPermission;
3440  }
3441
3442  public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
3443    this.ignorePreCheckPermission = ignorePreCheckPermission;
3444  }
3445
3446  /**
3447   * @param mm maximum number of regions to merge into a single region.
3448   */
3449  public void setMaxMerge(int mm) {
3450    this.maxMerge = mm;
3451  }
3452
3453  public int getMaxMerge() {
3454    return maxMerge;
3455  }
3456
3457  public void setMaxOverlapsToSideline(int mo) {
3458    this.maxOverlapsToSideline = mo;
3459  }
3460
3461  public int getMaxOverlapsToSideline() {
3462    return maxOverlapsToSideline;
3463  }
3464
3465  /**
3466   * Only check/fix tables specified by the list,
3467   * Empty list means all tables are included.
3468   */
3469  boolean isTableIncluded(TableName table) {
3470    return (tablesIncluded.isEmpty()) || tablesIncluded.contains(table);
3471  }
3472
3473  public void includeTable(TableName table) {
3474    tablesIncluded.add(table);
3475  }
3476
3477  Set<TableName> getIncludedTables() {
3478    return new HashSet<>(tablesIncluded);
3479  }
3480
3481  /**
3482   * We are interested in only those tables that have not changed their state in
3483   * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
3484   * @param seconds - the time in seconds
3485   */
3486  public void setTimeLag(long seconds) {
3487    timelag = seconds * 1000; // convert to milliseconds
3488  }
3489
3490  /**
3491   *
3492   * @param sidelineDir - HDFS path to sideline data
3493   */
3494  public void setSidelineDir(String sidelineDir) {
3495    this.sidelineDir = new Path(sidelineDir);
3496  }
3497
3498  protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
3499    return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
3500  }
3501
3502  public HFileCorruptionChecker getHFilecorruptionChecker() {
3503    return hfcc;
3504  }
3505
3506  public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
3507    this.hfcc = hfcc;
3508  }
3509
3510  public void setRetCode(int code) {
3511    this.retcode = code;
3512  }
3513
3514  public int getRetCode() {
3515    return retcode;
3516  }
3517
3518  protected HBaseFsck printUsageAndExit() {
3519    StringWriter sw = new StringWriter(2048);
3520    PrintWriter out = new PrintWriter(sw);
3521    out.println("");
3522    out.println("-----------------------------------------------------------------------");
3523    out.println("NOTE: As of HBase version 2.0, the hbck tool is significantly changed.");
3524    out.println("In general, all Read-Only options are supported and can be be used");
3525    out.println("safely. Most -fix/ -repair options are NOT supported. Please see usage");
3526    out.println("below for details on which options are not supported.");
3527    out.println("-----------------------------------------------------------------------");
3528    out.println("");
3529    out.println("Usage: fsck [opts] {only tables}");
3530    out.println(" where [opts] are:");
3531    out.println("   -help Display help options (this)");
3532    out.println("   -details Display full report of all regions.");
3533    out.println("   -timelag <timeInSeconds>  Process only regions that " +
3534                       " have not experienced any metadata updates in the last " +
3535                       " <timeInSeconds> seconds.");
3536    out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
3537        " before checking if the fix worked if run with -fix");
3538    out.println("   -summary Print only summary of the tables and status.");
3539    out.println("   -metaonly Only check the state of the hbase:meta table.");
3540    out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
3541    out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
3542    out.println("   -exclusive Abort if another hbck is exclusive or fixing.");
3543
3544    out.println("");
3545    out.println("  Datafile Repair options: (expert features, use with caution!)");
3546    out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
3547    out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
3548
3549    out.println("");
3550    out.println(" Replication options");
3551    out.println("   -fixReplication   Deletes replication queues for removed peers");
3552
3553    out.println("");
3554    out.println("  Metadata Repair options supported as of version 2.0: (expert features, use with caution!)");
3555    out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
3556    out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
3557    out.println("   -fixHFileLinks  Try to offline lingering HFileLinks");
3558    out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
3559        + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
3560    out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
3561
3562    out.println("");
3563    out.println("NOTE: Following options are NOT supported as of HBase version 2.0+.");
3564    out.println("");
3565    out.println("  UNSUPPORTED Metadata Repair options: (expert features, use with caution!)");
3566    out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
3567    out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
3568    out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
3569    out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
3570    out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
3571    out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
3572    out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
3573    out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
3574    out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
3575    out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
3576    out.println("   -fixSplitParents  Try to force offline split parents to be online.");
3577    out.println("   -removeParents    Try to offline and sideline lingering parents and keep daughter regions.");
3578    out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
3579        + " (empty REGIONINFO_QUALIFIER rows)");
3580
3581    out.println("");
3582    out.println("  UNSUPPORTED Metadata Repair shortcuts");
3583    out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
3584        "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles" +
3585        "-fixHFileLinks");
3586    out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
3587    out.println("");
3588    out.println(" Replication options");
3589    out.println("   -fixReplication   Deletes replication queues for removed peers");
3590    out.println("   -cleanReplicationBrarier [tableName] clean the replication barriers " +
3591        "of a specified table, tableName is required");
3592    out.flush();
3593    errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
3594
3595    setRetCode(-2);
3596    return this;
3597  }
3598
3599  /**
3600   * Main program
3601   *
3602   * @param args
3603   * @throws Exception
3604   */
3605  public static void main(String[] args) throws Exception {
3606    // create a fsck object
3607    Configuration conf = HBaseConfiguration.create();
3608    Path hbasedir = FSUtils.getRootDir(conf);
3609    URI defaultFs = hbasedir.getFileSystem(conf).getUri();
3610    FSUtils.setFsDefault(conf, new Path(defaultFs));
3611    int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
3612    System.exit(ret);
3613  }
3614
3615  /**
3616   * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
3617   */
3618  static class HBaseFsckTool extends Configured implements Tool {
3619    HBaseFsckTool(Configuration conf) { super(conf); }
3620    @Override
3621    public int run(String[] args) throws Exception {
3622      HBaseFsck hbck = new HBaseFsck(getConf());
3623      hbck.exec(hbck.executor, args);
3624      hbck.close();
3625      return hbck.getRetCode();
3626    }
3627  };
3628
3629
3630  public HBaseFsck exec(ExecutorService exec, String[] args)
3631      throws KeeperException, IOException, InterruptedException, ReplicationException {
3632    long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
3633
3634    boolean checkCorruptHFiles = false;
3635    boolean sidelineCorruptHFiles = false;
3636
3637    // Process command-line args.
3638    for (int i = 0; i < args.length; i++) {
3639      String cmd = args[i];
3640      if (cmd.equals("-help") || cmd.equals("-h")) {
3641        return printUsageAndExit();
3642      } else if (cmd.equals("-details")) {
3643        setDisplayFullReport();
3644      } else if (cmd.equals("-exclusive")) {
3645        setForceExclusive();
3646      } else if (cmd.equals("-timelag")) {
3647        if (i == args.length - 1) {
3648          errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
3649          return printUsageAndExit();
3650        }
3651        try {
3652          long timelag = Long.parseLong(args[++i]);
3653          setTimeLag(timelag);
3654        } catch (NumberFormatException e) {
3655          errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
3656          return printUsageAndExit();
3657        }
3658      } else if (cmd.equals("-sleepBeforeRerun")) {
3659        if (i == args.length - 1) {
3660          errors.reportError(ERROR_CODE.WRONG_USAGE,
3661            "HBaseFsck: -sleepBeforeRerun needs a value.");
3662          return printUsageAndExit();
3663        }
3664        try {
3665          sleepBeforeRerun = Long.parseLong(args[++i]);
3666        } catch (NumberFormatException e) {
3667          errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
3668          return printUsageAndExit();
3669        }
3670      } else if (cmd.equals("-sidelineDir")) {
3671        if (i == args.length - 1) {
3672          errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
3673          return printUsageAndExit();
3674        }
3675        setSidelineDir(args[++i]);
3676      } else if (cmd.equals("-fix")) {
3677        errors.reportError(ERROR_CODE.WRONG_USAGE,
3678          "This option is deprecated, please use  -fixAssignments instead.");
3679        setFixAssignments(true);
3680      } else if (cmd.equals("-fixAssignments")) {
3681        setFixAssignments(true);
3682      } else if (cmd.equals("-fixMeta")) {
3683        setFixMeta(true);
3684      } else if (cmd.equals("-noHdfsChecking")) {
3685        setCheckHdfs(false);
3686      } else if (cmd.equals("-fixHdfsHoles")) {
3687        setFixHdfsHoles(true);
3688      } else if (cmd.equals("-fixHdfsOrphans")) {
3689        setFixHdfsOrphans(true);
3690      } else if (cmd.equals("-fixTableOrphans")) {
3691        setFixTableOrphans(true);
3692      } else if (cmd.equals("-fixHdfsOverlaps")) {
3693        setFixHdfsOverlaps(true);
3694      } else if (cmd.equals("-fixVersionFile")) {
3695        setFixVersionFile(true);
3696      } else if (cmd.equals("-sidelineBigOverlaps")) {
3697        setSidelineBigOverlaps(true);
3698      } else if (cmd.equals("-fixSplitParents")) {
3699        setFixSplitParents(true);
3700      } else if (cmd.equals("-removeParents")) {
3701        setRemoveParents(true);
3702      } else if (cmd.equals("-ignorePreCheckPermission")) {
3703        setIgnorePreCheckPermission(true);
3704      } else if (cmd.equals("-checkCorruptHFiles")) {
3705        checkCorruptHFiles = true;
3706      } else if (cmd.equals("-sidelineCorruptHFiles")) {
3707        sidelineCorruptHFiles = true;
3708      } else if (cmd.equals("-fixReferenceFiles")) {
3709        setFixReferenceFiles(true);
3710      } else if (cmd.equals("-fixHFileLinks")) {
3711        setFixHFileLinks(true);
3712      } else if (cmd.equals("-fixEmptyMetaCells")) {
3713        setFixEmptyMetaCells(true);
3714      } else if (cmd.equals("-repair")) {
3715        // this attempts to merge overlapping hdfs regions, needs testing
3716        // under load
3717        setFixHdfsHoles(true);
3718        setFixHdfsOrphans(true);
3719        setFixMeta(true);
3720        setFixAssignments(true);
3721        setFixHdfsOverlaps(true);
3722        setFixVersionFile(true);
3723        setSidelineBigOverlaps(true);
3724        setFixSplitParents(false);
3725        setCheckHdfs(true);
3726        setFixReferenceFiles(true);
3727        setFixHFileLinks(true);
3728      } else if (cmd.equals("-repairHoles")) {
3729        // this will make all missing hdfs regions available but may lose data
3730        setFixHdfsHoles(true);
3731        setFixHdfsOrphans(false);
3732        setFixMeta(true);
3733        setFixAssignments(true);
3734        setFixHdfsOverlaps(false);
3735        setSidelineBigOverlaps(false);
3736        setFixSplitParents(false);
3737        setCheckHdfs(true);
3738      } else if (cmd.equals("-maxOverlapsToSideline")) {
3739        if (i == args.length - 1) {
3740          errors.reportError(ERROR_CODE.WRONG_USAGE,
3741            "-maxOverlapsToSideline needs a numeric value argument.");
3742          return printUsageAndExit();
3743        }
3744        try {
3745          int maxOverlapsToSideline = Integer.parseInt(args[++i]);
3746          setMaxOverlapsToSideline(maxOverlapsToSideline);
3747        } catch (NumberFormatException e) {
3748          errors.reportError(ERROR_CODE.WRONG_USAGE,
3749            "-maxOverlapsToSideline needs a numeric value argument.");
3750          return printUsageAndExit();
3751        }
3752      } else if (cmd.equals("-maxMerge")) {
3753        if (i == args.length - 1) {
3754          errors.reportError(ERROR_CODE.WRONG_USAGE,
3755            "-maxMerge needs a numeric value argument.");
3756          return printUsageAndExit();
3757        }
3758        try {
3759          int maxMerge = Integer.parseInt(args[++i]);
3760          setMaxMerge(maxMerge);
3761        } catch (NumberFormatException e) {
3762          errors.reportError(ERROR_CODE.WRONG_USAGE,
3763            "-maxMerge needs a numeric value argument.");
3764          return printUsageAndExit();
3765        }
3766      } else if (cmd.equals("-summary")) {
3767        setSummary();
3768      } else if (cmd.equals("-metaonly")) {
3769        setCheckMetaOnly();
3770      } else if (cmd.equals("-boundaries")) {
3771        setRegionBoundariesCheck();
3772      } else if (cmd.equals("-fixReplication")) {
3773        setFixReplication(true);
3774      } else if (cmd.equals("-cleanReplicationBarrier")) {
3775        setCleanReplicationBarrier(true);
3776        if(args[++i].startsWith("-")){
3777          printUsageAndExit();
3778        }
3779        setCleanReplicationBarrierTable(args[i]);
3780      } else if (cmd.startsWith("-")) {
3781        errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
3782        return printUsageAndExit();
3783      } else {
3784        includeTable(TableName.valueOf(cmd));
3785        errors.print("Allow checking/fixes for table: " + cmd);
3786      }
3787    }
3788
3789    errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
3790
3791    // pre-check current user has FS write permission or not
3792    try {
3793      preCheckPermission();
3794    } catch (AccessDeniedException ace) {
3795      Runtime.getRuntime().exit(-1);
3796    } catch (IOException ioe) {
3797      Runtime.getRuntime().exit(-1);
3798    }
3799
3800    // do the real work of hbck
3801    connect();
3802
3803    // after connecting to server above, we have server version
3804    // check if unsupported option is specified based on server version
3805    if (!isOptionsSupported(args)) {
3806      return printUsageAndExit();
3807    }
3808
3809    try {
3810      // if corrupt file mode is on, first fix them since they may be opened later
3811      if (checkCorruptHFiles || sidelineCorruptHFiles) {
3812        LOG.info("Checking all hfiles for corruption");
3813        HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
3814        setHFileCorruptionChecker(hfcc); // so we can get result
3815        Collection<TableName> tables = getIncludedTables();
3816        Collection<Path> tableDirs = new ArrayList<>();
3817        Path rootdir = FSUtils.getRootDir(getConf());
3818        if (tables.size() > 0) {
3819          for (TableName t : tables) {
3820            tableDirs.add(FSUtils.getTableDir(rootdir, t));
3821          }
3822        } else {
3823          tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
3824        }
3825        hfcc.checkTables(tableDirs);
3826        hfcc.report(errors);
3827      }
3828
3829      // check and fix table integrity, region consistency.
3830      int code = onlineHbck();
3831      setRetCode(code);
3832      // If we have changed the HBase state it is better to run hbck again
3833      // to see if we haven't broken something else in the process.
3834      // We run it only once more because otherwise we can easily fall into
3835      // an infinite loop.
3836      if (shouldRerun()) {
3837        try {
3838          LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
3839          Thread.sleep(sleepBeforeRerun);
3840        } catch (InterruptedException ie) {
3841          LOG.warn("Interrupted while sleeping");
3842          return this;
3843        }
3844        // Just report
3845        setFixAssignments(false);
3846        setFixMeta(false);
3847        setFixHdfsHoles(false);
3848        setFixHdfsOverlaps(false);
3849        setFixVersionFile(false);
3850        setFixTableOrphans(false);
3851        errors.resetErrors();
3852        code = onlineHbck();
3853        setRetCode(code);
3854      }
3855    } finally {
3856      IOUtils.closeQuietly(this);
3857    }
3858    return this;
3859  }
3860
3861  private boolean isOptionsSupported(String[] args) {
3862    boolean result = true;
3863    String hbaseServerVersion = status.getHBaseVersion();
3864    if (VersionInfo.compareVersion("2.any.any", hbaseServerVersion) < 0) {
3865      // Process command-line args.
3866      for (String arg : args) {
3867        if (unsupportedOptionsInV2.contains(arg)) {
3868          errors.reportError(ERROR_CODE.UNSUPPORTED_OPTION,
3869              "option '" + arg + "' is not " + "supportted!");
3870          result = false;
3871          break;
3872        }
3873      }
3874    }
3875    return result;
3876  }
3877
3878  public void setCleanReplicationBarrierTable(String cleanReplicationBarrierTable) {
3879    this.cleanReplicationBarrierTable = TableName.valueOf(cleanReplicationBarrierTable);
3880  }
3881
3882  public void cleanReplicationBarrier() throws IOException {
3883    if (!cleanReplicationBarrier || cleanReplicationBarrierTable == null) {
3884      return;
3885    }
3886    if (cleanReplicationBarrierTable.isSystemTable()) {
3887      errors.reportError(ERROR_CODE.INVALID_TABLE,
3888        "invalid table: " + cleanReplicationBarrierTable);
3889      return;
3890    }
3891
3892    boolean isGlobalScope = false;
3893    try {
3894      isGlobalScope = admin.getDescriptor(cleanReplicationBarrierTable).hasGlobalReplicationScope();
3895    } catch (TableNotFoundException e) {
3896      LOG.info("we may need to clean some erroneous data due to bugs");
3897    }
3898
3899    if (isGlobalScope) {
3900      errors.reportError(ERROR_CODE.INVALID_TABLE,
3901        "table's replication scope is global: " + cleanReplicationBarrierTable);
3902      return;
3903    }
3904    List<byte[]> regionNames = new ArrayList<>();
3905    Scan barrierScan = new Scan();
3906    barrierScan.setCaching(100);
3907    barrierScan.addFamily(HConstants.REPLICATION_BARRIER_FAMILY);
3908    barrierScan
3909        .withStartRow(MetaTableAccessor.getTableStartRowForMeta(cleanReplicationBarrierTable,
3910          MetaTableAccessor.QueryType.REGION))
3911        .withStopRow(MetaTableAccessor.getTableStopRowForMeta(cleanReplicationBarrierTable,
3912          MetaTableAccessor.QueryType.REGION));
3913    Result result;
3914    try (ResultScanner scanner = meta.getScanner(barrierScan)) {
3915      while ((result = scanner.next()) != null) {
3916        regionNames.add(result.getRow());
3917      }
3918    }
3919    if (regionNames.size() <= 0) {
3920      errors.reportError(ERROR_CODE.INVALID_TABLE,
3921        "there is no barriers of this table: " + cleanReplicationBarrierTable);
3922      return;
3923    }
3924    ReplicationQueueStorage queueStorage =
3925        ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
3926    List<ReplicationPeerDescription> peerDescriptions = admin.listReplicationPeers();
3927    if (peerDescriptions != null && peerDescriptions.size() > 0) {
3928      List<String> peers = peerDescriptions.stream()
3929          .filter(peerConfig -> peerConfig.getPeerConfig()
3930            .needToReplicate(cleanReplicationBarrierTable))
3931          .map(peerConfig -> peerConfig.getPeerId()).collect(Collectors.toList());
3932      try {
3933        List<String> batch = new ArrayList<>();
3934        for (String peer : peers) {
3935          for (byte[] regionName : regionNames) {
3936            batch.add(RegionInfo.encodeRegionName(regionName));
3937            if (batch.size() % 100 == 0) {
3938              queueStorage.removeLastSequenceIds(peer, batch);
3939              batch.clear();
3940            }
3941          }
3942          if (batch.size() > 0) {
3943            queueStorage.removeLastSequenceIds(peer, batch);
3944            batch.clear();
3945          }
3946        }
3947      } catch (ReplicationException re) {
3948        throw new IOException(re);
3949      }
3950    }
3951    for (byte[] regionName : regionNames) {
3952      meta.delete(new Delete(regionName).addFamily(HConstants.REPLICATION_BARRIER_FAMILY));
3953    }
3954    setShouldRerun();
3955  }
3956
3957  /**
3958   * ls -r for debugging purposes
3959   */
3960  void debugLsr(Path p) throws IOException {
3961    debugLsr(getConf(), p, errors);
3962  }
3963
3964  /**
3965   * ls -r for debugging purposes
3966   */
3967  public static void debugLsr(Configuration conf,
3968      Path p) throws IOException {
3969    debugLsr(conf, p, new PrintingErrorReporter());
3970  }
3971
3972  /**
3973   * ls -r for debugging purposes
3974   */
3975  public static void debugLsr(Configuration conf,
3976      Path p, HbckErrorReporter errors) throws IOException {
3977    if (!LOG.isDebugEnabled() || p == null) {
3978      return;
3979    }
3980    FileSystem fs = p.getFileSystem(conf);
3981
3982    if (!fs.exists(p)) {
3983      // nothing
3984      return;
3985    }
3986    errors.print(p.toString());
3987
3988    if (fs.isFile(p)) {
3989      return;
3990    }
3991
3992    if (fs.getFileStatus(p).isDirectory()) {
3993      FileStatus[] fss= fs.listStatus(p);
3994      for (FileStatus status : fss) {
3995        debugLsr(conf, status.getPath(), errors);
3996      }
3997    }
3998  }
3999}