View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.Closeable;
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.io.InterruptedIOException;
24  import java.io.PrintWriter;
25  import java.io.StringWriter;
26  import java.net.InetAddress;
27  import java.net.URI;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collection;
31  import java.util.Collections;
32  import java.util.Comparator;
33  import java.util.HashMap;
34  import java.util.HashSet;
35  import java.util.Iterator;
36  import java.util.List;
37  import java.util.Map;
38  import java.util.Map.Entry;
39  import java.util.Set;
40  import java.util.SortedMap;
41  import java.util.SortedSet;
42  import java.util.TreeMap;
43  import java.util.TreeSet;
44  import java.util.concurrent.Callable;
45  import java.util.concurrent.ConcurrentSkipListMap;
46  import java.util.concurrent.ExecutionException;
47  import java.util.concurrent.ExecutorService;
48  import java.util.concurrent.Executors;
49  import java.util.concurrent.Future;
50  import java.util.concurrent.FutureTask;
51  import java.util.concurrent.ScheduledThreadPoolExecutor;
52  import java.util.concurrent.TimeUnit;
53  import java.util.concurrent.TimeoutException;
54  import java.util.concurrent.atomic.AtomicBoolean;
55  import java.util.concurrent.atomic.AtomicInteger;
56  
57  import com.google.common.base.Joiner;
58  import com.google.common.base.Preconditions;
59  import com.google.common.collect.ImmutableList;
60  import com.google.common.collect.Lists;
61  import com.google.common.collect.Multimap;
62  import com.google.common.collect.Ordering;
63  import com.google.common.collect.TreeMultimap;
64  import com.google.protobuf.ServiceException;
65  import org.apache.commons.io.IOUtils;
66  import org.apache.commons.lang.RandomStringUtils;
67  import org.apache.commons.lang.StringUtils;
68  import org.apache.commons.logging.Log;
69  import org.apache.commons.logging.LogFactory;
70  import org.apache.hadoop.conf.Configuration;
71  import org.apache.hadoop.conf.Configured;
72  import org.apache.hadoop.fs.FSDataOutputStream;
73  import org.apache.hadoop.fs.FileStatus;
74  import org.apache.hadoop.fs.FileSystem;
75  import org.apache.hadoop.fs.Path;
76  import org.apache.hadoop.fs.permission.FsAction;
77  import org.apache.hadoop.fs.permission.FsPermission;
78  import org.apache.hadoop.hbase.Abortable;
79  import org.apache.hadoop.hbase.Cell;
80  import org.apache.hadoop.hbase.CellUtil;
81  import org.apache.hadoop.hbase.ClusterStatus;
82  import org.apache.hadoop.hbase.HBaseConfiguration;
83  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
84  import org.apache.hadoop.hbase.HColumnDescriptor;
85  import org.apache.hadoop.hbase.HConstants;
86  import org.apache.hadoop.hbase.HRegionInfo;
87  import org.apache.hadoop.hbase.HRegionLocation;
88  import org.apache.hadoop.hbase.HTableDescriptor;
89  import org.apache.hadoop.hbase.KeyValue;
90  import org.apache.hadoop.hbase.MasterNotRunningException;
91  import org.apache.hadoop.hbase.MetaTableAccessor;
92  import org.apache.hadoop.hbase.RegionLocations;
93  import org.apache.hadoop.hbase.ServerName;
94  import org.apache.hadoop.hbase.TableDescriptor;
95  import org.apache.hadoop.hbase.TableName;
96  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
97  import org.apache.hadoop.hbase.classification.InterfaceAudience;
98  import org.apache.hadoop.hbase.classification.InterfaceStability;
99  import org.apache.hadoop.hbase.client.Admin;
100 import org.apache.hadoop.hbase.client.ClusterConnection;
101 import org.apache.hadoop.hbase.client.Connection;
102 import org.apache.hadoop.hbase.client.ConnectionFactory;
103 import org.apache.hadoop.hbase.client.Delete;
104 import org.apache.hadoop.hbase.client.Get;
105 import org.apache.hadoop.hbase.client.HConnection;
106 import org.apache.hadoop.hbase.client.Put;
107 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
108 import org.apache.hadoop.hbase.client.Result;
109 import org.apache.hadoop.hbase.client.RowMutations;
110 import org.apache.hadoop.hbase.client.Table;
111 import org.apache.hadoop.hbase.client.TableState;
112 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
113 import org.apache.hadoop.hbase.io.hfile.HFile;
114 import org.apache.hadoop.hbase.master.MasterFileSystem;
115 import org.apache.hadoop.hbase.master.RegionState;
116 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
117 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
118 import org.apache.hadoop.hbase.regionserver.HRegion;
119 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
120 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
121 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
122 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
123 import org.apache.hadoop.hbase.security.AccessDeniedException;
124 import org.apache.hadoop.hbase.security.UserProvider;
125 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
126 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
127 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
128 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
129 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
130 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
131 import org.apache.hadoop.hbase.wal.WAL;
132 import org.apache.hadoop.hbase.wal.WALFactory;
133 import org.apache.hadoop.hbase.wal.WALSplitter;
134 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
135 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
136 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
137 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
138 import org.apache.hadoop.ipc.RemoteException;
139 import org.apache.hadoop.security.UserGroupInformation;
140 import org.apache.hadoop.util.ReflectionUtils;
141 import org.apache.hadoop.util.Tool;
142 import org.apache.hadoop.util.ToolRunner;
143 import org.apache.zookeeper.KeeperException;
144 
145 /**
146  * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
147  * table integrity problems in a corrupted HBase.
148  * <p>
149  * Region consistency checks verify that hbase:meta, region deployment on region
150  * servers and the state of data in HDFS (.regioninfo files) all are in
151  * accordance.
152  * <p>
153  * Table integrity checks verify that all possible row keys resolve to exactly
154  * one region of a table.  This means there are no individual degenerate
155  * or backwards regions; no holes between regions; and that there are no
156  * overlapping regions.
157  * <p>
158  * The general repair strategy works in two phases:
159  * <ol>
160  * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
161  * <li> Repair Region Consistency with hbase:meta and assignments
162  * </ol>
163  * <p>
164  * For table integrity repairs, the tables' region directories are scanned
165  * for .regioninfo files.  Each table's integrity is then verified.  If there
166  * are any orphan regions (regions with no .regioninfo files) or holes, new
167  * regions are fabricated.  Backwards regions are sidelined as well as empty
168  * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
169  * a new region is created and all data is merged into the new region.
170  * <p>
171  * Table integrity repairs deal solely with HDFS and could potentially be done
172  * offline -- the hbase region servers or master do not need to be running.
173  * This phase can eventually be used to completely reconstruct the hbase:meta table in
174  * an offline fashion.
175  * <p>
176  * Region consistency requires three conditions -- 1) valid .regioninfo file
177  * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
178  * and 3) a region is deployed only at the regionserver that was assigned to
179  * with proper state in the master.
180  * <p>
181  * Region consistency repairs require hbase to be online so that hbck can
182  * contact the HBase master and region servers.  The hbck#connect() method must
183  * first be called successfully.  Much of the region consistency information
184  * is transient and less risky to repair.
185  * <p>
186  * If hbck is run from the command line, there are a handful of arguments that
187  * can be used to limit the kinds of repairs hbck will do.  See the code in
188  * {@link #printUsageAndExit()} for more details.
189  */
190 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
191 @InterfaceStability.Evolving
192 public class HBaseFsck extends Configured implements Closeable {
193   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
194   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
195   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
196   private static boolean rsSupportsOffline = true;
197   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
198   private static final int DEFAULT_MAX_MERGE = 5;
199   private static final String TO_BE_LOADED = "to_be_loaded";
200   private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
201   private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5;
202   private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds
203   private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds
204   // We have to set the timeout value > HdfsConstants.LEASE_SOFTLIMIT_PERIOD.
205   // In HADOOP-2.6 and later, the Namenode proxy now created with custom RetryPolicy for
206   // AlreadyBeingCreatedException which is implies timeout on this operations up to
207   // HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds).
208   private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds
209 
210   /**********************
211    * Internal resources
212    **********************/
213   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
214   private ClusterStatus status;
215   private ClusterConnection connection;
216   private Admin admin;
217   private Table meta;
218   // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
219   protected ExecutorService executor;
220   private long startMillis = EnvironmentEdgeManager.currentTime();
221   private HFileCorruptionChecker hfcc;
222   private int retcode = 0;
223   private Path HBCK_LOCK_PATH;
224   private FSDataOutputStream hbckOutFd;
225   // This lock is to prevent cleanup of balancer resources twice between
226   // ShutdownHook and the main code. We cleanup only if the connect() is
227   // successful
228   private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
229 
230   /***********
231    * Options
232    ***********/
233   private static boolean details = false; // do we display the full report
234   private static boolean useLock = true; // do we use the hbck exclusivity lock
235   private static boolean switchBalancer = true; // do we turn the balancer off while running
236   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
237   private boolean fixAssignments = false; // fix assignment errors?
238   private boolean fixMeta = false; // fix meta errors?
239   private boolean checkHdfs = true; // load and check fs consistency?
240   private boolean fixHdfsHoles = false; // fix fs holes?
241   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
242   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
243   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
244   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
245   private boolean fixSplitParents = false; // fix lingering split parents
246   private boolean fixReferenceFiles = false; // fix lingering reference store file
247   private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
248   private boolean fixTableLocks = false; // fix table locks which are expired
249   private boolean fixAny = false; // Set to true if any of the fix is required.
250 
251   // limit checking/fixes to listed tables, if empty attempt to check/fix all
252   // hbase:meta are always checked
253   private Set<TableName> tablesIncluded = new HashSet<TableName>();
254   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
255   // maximum number of overlapping regions to sideline
256   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE;
257   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
258   private Path sidelineDir = null;
259 
260   private boolean rerun = false; // if we tried to fix something, rerun hbck
261   private static boolean summary = false; // if we want to print less output
262   private boolean checkMetaOnly = false;
263   private boolean checkRegionBoundaries = false;
264   private boolean ignorePreCheckPermission = false; // if pre-check permission
265 
266   /*********
267    * State
268    *********/
269   final private ErrorReporter errors;
270   int fixes = 0;
271 
272   /**
273    * This map contains the state of all hbck items.  It maps from encoded region
274    * name to HbckInfo structure.  The information contained in HbckInfo is used
275    * to detect and correct consistency (hdfs/meta/deployment) problems.
276    */
277   private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
278   // Empty regioninfo qualifiers in hbase:meta
279   private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
280 
281   /**
282    * This map from Tablename -> TableInfo contains the structures necessary to
283    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
284    * to prevent dupes.
285    *
286    * If tablesIncluded is empty, this map contains all tables.
287    * Otherwise, it contains only meta tables and tables in tablesIncluded,
288    * unless checkMetaOnly is specified, in which case, it contains only
289    * the meta table
290    */
291   private SortedMap<TableName, TableInfo> tablesInfo =
292       new ConcurrentSkipListMap<TableName, TableInfo>();
293 
294   /**
295    * When initially looking at HDFS, we attempt to find any orphaned data.
296    */
297   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
298 
299   private Map<TableName, Set<String>> orphanTableDirs =
300       new HashMap<TableName, Set<String>>();
301   private Map<TableName, TableState> tableStates =
302       new HashMap<TableName, TableState>();
303   private final RetryCounterFactory lockFileRetryCounterFactory;
304 
305   private Map<TableName, Set<String>> skippedRegions = new HashMap<TableName, Set<String>>();
306 
307   /**
308    * Constructor
309    *
310    * @param conf Configuration object
311    * @throws MasterNotRunningException if the master is not running
312    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
313    */
314   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
315       ZooKeeperConnectionException, IOException, ClassNotFoundException {
316     super(conf);
317     // make a copy, just to be sure we're not overriding someone else's config
318     setConf(HBaseConfiguration.create(getConf()));
319     // disable blockcache for tool invocation, see HBASE-10500
320     getConf().setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0);
321     // Disable usage of meta replicas in hbck
322     getConf().setBoolean(HConstants.USE_META_REPLICAS, false);
323     errors = getErrorReporter(conf);
324 
325     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
326     executor = new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
327     lockFileRetryCounterFactory = new RetryCounterFactory(
328       getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
329       getConf().getInt(
330         "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),
331       getConf().getInt(
332         "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));
333   }
334 
335   /**
336    * Constructor
337    *
338    * @param conf
339    *          Configuration object
340    * @throws MasterNotRunningException
341    *           if the master is not running
342    * @throws ZooKeeperConnectionException
343    *           if unable to connect to ZooKeeper
344    */
345   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
346       ZooKeeperConnectionException, IOException, ClassNotFoundException {
347     super(conf);
348     errors = getErrorReporter(getConf());
349     this.executor = exec;
350     lockFileRetryCounterFactory = new RetryCounterFactory(
351       getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
352       getConf().getInt(
353         "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),
354       getConf().getInt(
355         "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));
356   }
357 
358   private class FileLockCallable implements Callable<FSDataOutputStream> {
359     RetryCounter retryCounter;
360 
361     public FileLockCallable(RetryCounter retryCounter) {
362       this.retryCounter = retryCounter;
363     }
364     @Override
365     public FSDataOutputStream call() throws IOException {
366       try {
367         FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
368         FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
369             HConstants.DATA_FILE_UMASK_KEY);
370         Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
371         fs.mkdirs(tmpDir);
372         HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
373         final FSDataOutputStream out = createFileWithRetries(fs, HBCK_LOCK_PATH, defaultPerms);
374         out.writeBytes(InetAddress.getLocalHost().toString());
375         out.flush();
376         return out;
377       } catch(RemoteException e) {
378         if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
379           return null;
380         } else {
381           throw e;
382         }
383       }
384     }
385 
386     private FSDataOutputStream createFileWithRetries(final FileSystem fs,
387         final Path hbckLockFilePath, final FsPermission defaultPerms)
388         throws IOException {
389 
390       IOException exception = null;
391       do {
392         try {
393           return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false);
394         } catch (IOException ioe) {
395           LOG.info("Failed to create lock file " + hbckLockFilePath.getName()
396               + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of "
397               + retryCounter.getMaxAttempts());
398           LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(),
399               ioe);
400           try {
401             exception = ioe;
402             retryCounter.sleepUntilNextRetry();
403           } catch (InterruptedException ie) {
404             throw (InterruptedIOException) new InterruptedIOException(
405                 "Can't create lock file " + hbckLockFilePath.getName())
406             .initCause(ie);
407           }
408         }
409       } while (retryCounter.shouldRetry());
410 
411       throw exception;
412     }
413   }
414 
415   /**
416    * This method maintains a lock using a file. If the creation fails we return null
417    *
418    * @return FSDataOutputStream object corresponding to the newly opened lock file
419    * @throws IOException
420    */
421   private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
422     RetryCounter retryCounter = lockFileRetryCounterFactory.create();
423     FileLockCallable callable = new FileLockCallable(retryCounter);
424     ExecutorService executor = Executors.newFixedThreadPool(1);
425     FutureTask<FSDataOutputStream> futureTask = new FutureTask<FSDataOutputStream>(callable);
426     executor.execute(futureTask);
427     final int timeoutInSeconds = getConf().getInt(
428       "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT);
429     FSDataOutputStream stream = null;
430     try {
431       stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);
432     } catch (ExecutionException ee) {
433       LOG.warn("Encountered exception when opening lock file", ee);
434     } catch (InterruptedException ie) {
435       LOG.warn("Interrupted when opening lock file", ie);
436       Thread.currentThread().interrupt();
437     } catch (TimeoutException exception) {
438       // took too long to obtain lock
439       LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");
440       futureTask.cancel(true);
441     } finally {
442       executor.shutdownNow();
443     }
444     return stream;
445   }
446 
447   private void unlockHbck() {
448     if (hbckLockCleanup.compareAndSet(true, false)) {
449       RetryCounter retryCounter = lockFileRetryCounterFactory.create();
450       do {
451         try {
452           IOUtils.closeQuietly(hbckOutFd);
453           FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()),
454               HBCK_LOCK_PATH, true);
455           LOG.info("Finishing hbck");
456           return;
457         } catch (IOException ioe) {
458           LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try="
459               + (retryCounter.getAttemptTimes() + 1) + " of "
460               + retryCounter.getMaxAttempts());
461           LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe);
462           try {
463             retryCounter.sleepUntilNextRetry();
464           } catch (InterruptedException ie) {
465             Thread.currentThread().interrupt();
466             LOG.warn("Interrupted while deleting lock file" +
467                 HBCK_LOCK_PATH);
468             return;
469           }
470         }
471       } while (retryCounter.shouldRetry());
472     }
473   }
474 
475   /**
476    * To repair region consistency, one must call connect() in order to repair
477    * online state.
478    */
479   public void connect() throws IOException {
480 
481     if (useLock) {
482       // Check if another instance of balancer is running
483       hbckOutFd = checkAndMarkRunningHbck();
484       if (hbckOutFd == null) {
485         setRetCode(-1);
486         LOG.error("Another instance of hbck is running, exiting this instance.[If you are sure" +
487             " no other instance is running, delete the lock file " +
488             HBCK_LOCK_PATH + " and rerun the tool]");
489         throw new IOException("Duplicate hbck - Abort");
490       }
491 
492       // Make sure to cleanup the lock
493       hbckLockCleanup.set(true);
494     }
495 
496 
497     // Add a shutdown hook to this thread, in case user tries to
498     // kill the hbck with a ctrl-c, we want to cleanup the lock so that
499     // it is available for further calls
500     Runtime.getRuntime().addShutdownHook(new Thread() {
501       @Override
502       public void run() {
503         IOUtils.closeQuietly(HBaseFsck.this);
504         unlockHbck();
505       }
506     });
507 
508     LOG.info("Launching hbck");
509 
510     connection = (ClusterConnection)ConnectionFactory.createConnection(getConf());
511     admin = connection.getAdmin();
512     meta = connection.getTable(TableName.META_TABLE_NAME);
513     status = admin.getClusterStatus();
514   }
515 
516   /**
517    * Get deployed regions according to the region servers.
518    */
519   private void loadDeployedRegions() throws IOException, InterruptedException {
520     // From the master, get a list of all known live region servers
521     Collection<ServerName> regionServers = status.getServers();
522     errors.print("Number of live region servers: " + regionServers.size());
523     if (details) {
524       for (ServerName rsinfo: regionServers) {
525         errors.print("  " + rsinfo.getServerName());
526       }
527     }
528 
529     // From the master, get a list of all dead region servers
530     Collection<ServerName> deadRegionServers = status.getDeadServerNames();
531     errors.print("Number of dead region servers: " + deadRegionServers.size());
532     if (details) {
533       for (ServerName name: deadRegionServers) {
534         errors.print("  " + name);
535       }
536     }
537 
538     // Print the current master name and state
539     errors.print("Master: " + status.getMaster());
540 
541     // Print the list of all backup masters
542     Collection<ServerName> backupMasters = status.getBackupMasters();
543     errors.print("Number of backup masters: " + backupMasters.size());
544     if (details) {
545       for (ServerName name: backupMasters) {
546         errors.print("  " + name);
547       }
548     }
549 
550     errors.print("Average load: " + status.getAverageLoad());
551     errors.print("Number of requests: " + status.getRequestsCount());
552     errors.print("Number of regions: " + status.getRegionsCount());
553 
554     Map<String, RegionState> rits = status.getRegionsInTransition();
555     errors.print("Number of regions in transition: " + rits.size());
556     if (details) {
557       for (RegionState state: rits.values()) {
558         errors.print("  " + state.toDescriptiveString());
559       }
560     }
561 
562     // Determine what's deployed
563     processRegionServers(regionServers);
564   }
565 
566   /**
567    * Clear the current state of hbck.
568    */
569   private void clearState() {
570     // Make sure regionInfo is empty before starting
571     fixes = 0;
572     regionInfoMap.clear();
573     emptyRegionInfoQualifiers.clear();
574     tableStates.clear();
575     errors.clear();
576     tablesInfo.clear();
577     orphanHdfsDirs.clear();
578     skippedRegions.clear();
579   }
580 
581   /**
582    * This repair method analyzes hbase data in hdfs and repairs it to satisfy
583    * the table integrity rules.  HBase doesn't need to be online for this
584    * operation to work.
585    */
586   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
587     // Initial pass to fix orphans.
588     if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
589         || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
590       LOG.info("Loading regioninfos HDFS");
591       // if nothing is happening this should always complete in two iterations.
592       int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
593       int curIter = 0;
594       do {
595         clearState(); // clears hbck state and reset fixes to 0 and.
596         // repair what's on HDFS
597         restoreHdfsIntegrity();
598         curIter++;// limit the number of iterations.
599       } while (fixes > 0 && curIter <= maxIterations);
600 
601       // Repairs should be done in the first iteration and verification in the second.
602       // If there are more than 2 passes, something funny has happened.
603       if (curIter > 2) {
604         if (curIter == maxIterations) {
605           LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
606               + "Tables integrity may not be fully repaired!");
607         } else {
608           LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
609         }
610       }
611     }
612   }
613 
614   /**
615    * This repair method requires the cluster to be online since it contacts
616    * region servers and the masters.  It makes each region's state in HDFS, in
617    * hbase:meta, and deployments consistent.
618    *
619    * @return If &gt; 0 , number of errors detected, if &lt; 0 there was an unrecoverable
620    * error.  If 0, we have a clean hbase.
621    */
622   public int onlineConsistencyRepair() throws IOException, KeeperException,
623     InterruptedException {
624     clearState();
625 
626     // get regions according to what is online on each RegionServer
627     loadDeployedRegions();
628     // check whether hbase:meta is deployed and online
629     recordMetaRegion();
630     // Check if hbase:meta is found only once and in the right place
631     if (!checkMetaRegion()) {
632       String errorMsg = "hbase:meta table is not consistent. ";
633       if (shouldFixAssignments()) {
634         errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
635       } else {
636         errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
637       }
638       errors.reportError(errorMsg + " Exiting...");
639       return -2;
640     }
641     // Not going with further consistency check for tables when hbase:meta itself is not consistent.
642     LOG.info("Loading regionsinfo from the hbase:meta table");
643     boolean success = loadMetaEntries();
644     if (!success) return -1;
645 
646     // Empty cells in hbase:meta?
647     reportEmptyMetaCells();
648 
649     // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
650     if (shouldFixEmptyMetaCells()) {
651       fixEmptyMetaCells();
652     }
653 
654     // get a list of all tables that have not changed recently.
655     if (!checkMetaOnly) {
656       reportTablesInFlux();
657     }
658 
659     // Get disabled tables states
660     loadTableStates();
661 
662     // load regiondirs and regioninfos from HDFS
663     if (shouldCheckHdfs()) {
664       LOG.info("Loading region directories from HDFS");
665       loadHdfsRegionDirs();
666       LOG.info("Loading region information from HDFS");
667       loadHdfsRegionInfos();
668     }
669 
670     // fix the orphan tables
671     fixOrphanTables();
672 
673     LOG.info("Checking and fixing region consistency");
674     // Check and fix consistency
675     checkAndFixConsistency();
676 
677     // Check integrity (does not fix)
678     checkIntegrity();
679     return errors.getErrorList().size();
680   }
681 
682   /**
683    * Contacts the master and prints out cluster-wide information
684    * @return 0 on success, non-zero on failure
685    */
686   public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
687     // print hbase server version
688     errors.print("Version: " + status.getHBaseVersion());
689     offlineHdfsIntegrityRepair();
690 
691     boolean oldBalancer = true;
692     // turn the balancer off
693     if (switchBalancer) {
694       oldBalancer = admin.setBalancerRunning(false, true);
695     }
696 
697     try {
698       onlineConsistencyRepair();
699     }
700     finally {
701       if (switchBalancer) {
702         admin.setBalancerRunning(oldBalancer, false);
703       }
704     }
705 
706     if (checkRegionBoundaries) {
707       checkRegionBoundaries();
708     }
709 
710     offlineReferenceFileRepair();
711 
712     checkAndFixTableLocks();
713 
714     // Remove the hbck lock
715     unlockHbck();
716 
717     // Print table summary
718     printTableSummary(tablesInfo);
719     return errors.summarize();
720   }
721 
722   public static byte[] keyOnly (byte[] b) {
723     if (b == null)
724       return b;
725     int rowlength = Bytes.toShort(b, 0);
726     byte[] result = new byte[rowlength];
727     System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
728     return result;
729   }
730 
731   @Override
732   public void close() throws IOException {
733     try {
734       unlockHbck();
735     } catch (Exception io) {
736       LOG.warn(io);
737     } finally {
738       IOUtils.closeQuietly(admin);
739       IOUtils.closeQuietly(meta);
740       IOUtils.closeQuietly(connection);
741     }
742   }
743 
744   private static class RegionBoundariesInformation {
745     public byte [] regionName;
746     public byte [] metaFirstKey;
747     public byte [] metaLastKey;
748     public byte [] storesFirstKey;
749     public byte [] storesLastKey;
750     @Override
751     public String toString () {
752       return "regionName=" + Bytes.toStringBinary(regionName) +
753              "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
754              "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
755              "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
756              "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
757     }
758   }
759 
760   public void checkRegionBoundaries() {
761     try {
762       ByteArrayComparator comparator = new ByteArrayComparator();
763       List<HRegionInfo> regions = MetaTableAccessor.getAllRegions(connection, true);
764       final RegionBoundariesInformation currentRegionBoundariesInformation =
765           new RegionBoundariesInformation();
766       Path hbaseRoot = FSUtils.getRootDir(getConf());
767       for (HRegionInfo regionInfo : regions) {
768         Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
769         currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
770         // For each region, get the start and stop key from the META and compare them to the
771         // same information from the Stores.
772         Path path = new Path(tableDir, regionInfo.getEncodedName());
773         FileSystem fs = path.getFileSystem(getConf());
774         FileStatus[] files = fs.listStatus(path);
775         // For all the column families in this region...
776         byte[] storeFirstKey = null;
777         byte[] storeLastKey = null;
778         for (FileStatus file : files) {
779           String fileName = file.getPath().toString();
780           fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
781           if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
782             FileStatus[] storeFiles = fs.listStatus(file.getPath());
783             // For all the stores in this column family.
784             for (FileStatus storeFile : storeFiles) {
785               HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
786                   getConf()), getConf());
787               if ((reader.getFirstKey() != null)
788                   && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
789                       ((KeyValue.KeyOnlyKeyValue) reader.getFirstKey()).getKey()) > 0))) {
790                 storeFirstKey = ((KeyValue.KeyOnlyKeyValue)reader.getFirstKey()).getKey();
791               }
792               if ((reader.getLastKey() != null)
793                   && ((storeLastKey == null) || (comparator.compare(storeLastKey,
794                       ((KeyValue.KeyOnlyKeyValue)reader.getLastKey()).getKey())) < 0)) {
795                 storeLastKey = ((KeyValue.KeyOnlyKeyValue)reader.getLastKey()).getKey();
796               }
797               reader.close();
798             }
799           }
800         }
801         currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
802         currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
803         currentRegionBoundariesInformation.storesFirstKey = storeFirstKey;
804         currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
805         if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
806           currentRegionBoundariesInformation.metaFirstKey = null;
807         if (currentRegionBoundariesInformation.metaLastKey.length == 0)
808           currentRegionBoundariesInformation.metaLastKey = null;
809 
810         // For a region to be correct, we need the META start key to be smaller or equal to the
811         // smallest start key from all the stores, and the start key from the next META entry to
812         // be bigger than the last key from all the current stores. First region start key is null;
813         // Last region end key is null; some regions can be empty and not have any store.
814 
815         boolean valid = true;
816         // Checking start key.
817         if ((currentRegionBoundariesInformation.storesFirstKey != null)
818             && (currentRegionBoundariesInformation.metaFirstKey != null)) {
819           valid = valid
820               && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
821                 currentRegionBoundariesInformation.metaFirstKey) >= 0;
822         }
823         // Checking stop key.
824         if ((currentRegionBoundariesInformation.storesLastKey != null)
825             && (currentRegionBoundariesInformation.metaLastKey != null)) {
826           valid = valid
827               && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
828                 currentRegionBoundariesInformation.metaLastKey) < 0;
829         }
830         if (!valid) {
831           errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
832             tablesInfo.get(regionInfo.getTable()));
833           LOG.warn("Region's boundaries not alligned between stores and META for:");
834           LOG.warn(currentRegionBoundariesInformation);
835         }
836       }
837     } catch (IOException e) {
838       LOG.error(e);
839     }
840   }
841 
842   /**
843    * Iterates through the list of all orphan/invalid regiondirs.
844    */
845   private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
846     for (HbckInfo hi : orphanHdfsDirs) {
847       LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
848       adoptHdfsOrphan(hi);
849     }
850   }
851 
852   /**
853    * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
854    * these orphans by creating a new region, and moving the column families,
855    * recovered edits, WALs, into the new region dir.  We determine the region
856    * startkey and endkeys by looking at all of the hfiles inside the column
857    * families to identify the min and max keys. The resulting region will
858    * likely violate table integrity but will be dealt with by merging
859    * overlapping regions.
860    */
861   @SuppressWarnings("deprecation")
862   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
863     Path p = hi.getHdfsRegionDir();
864     FileSystem fs = p.getFileSystem(getConf());
865     FileStatus[] dirs = fs.listStatus(p);
866     if (dirs == null) {
867       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
868           p + ". This dir could probably be deleted.");
869       return ;
870     }
871 
872     TableName tableName = hi.getTableName();
873     TableInfo tableInfo = tablesInfo.get(tableName);
874     Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
875     HTableDescriptor template = tableInfo.getHTD();
876 
877     // find min and max key values
878     Pair<byte[],byte[]> orphanRegionRange = null;
879     for (FileStatus cf : dirs) {
880       String cfName= cf.getPath().getName();
881       // TODO Figure out what the special dirs are
882       if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
883 
884       FileStatus[] hfiles = fs.listStatus(cf.getPath());
885       for (FileStatus hfile : hfiles) {
886         byte[] start, end;
887         HFile.Reader hf = null;
888         try {
889           CacheConfig cacheConf = new CacheConfig(getConf());
890           hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
891           hf.loadFileInfo();
892           Cell startKv = hf.getFirstKey();
893           start = CellUtil.cloneRow(startKv);
894           Cell endKv = hf.getLastKey();
895           end = CellUtil.cloneRow(endKv);
896         } catch (IOException ioe) {
897           LOG.warn("Problem reading orphan file " + hfile + ", skipping");
898           continue;
899         } catch (NullPointerException ioe) {
900           LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
901           continue;
902         } finally {
903           if (hf != null) {
904             hf.close();
905           }
906         }
907 
908         // expand the range to include the range of all hfiles
909         if (orphanRegionRange == null) {
910           // first range
911           orphanRegionRange = new Pair<byte[], byte[]>(start, end);
912         } else {
913           // TODO add test
914 
915           // expand range only if the hfile is wider.
916           if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
917             orphanRegionRange.setFirst(start);
918           }
919           if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
920             orphanRegionRange.setSecond(end);
921           }
922         }
923       }
924     }
925     if (orphanRegionRange == null) {
926       LOG.warn("No data in dir " + p + ", sidelining data");
927       fixes++;
928       sidelineRegionDir(fs, hi);
929       return;
930     }
931     LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
932         Bytes.toString(orphanRegionRange.getSecond()) + ")");
933 
934     // create new region on hdfs.  move data into place.
935     HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond());
936     LOG.info("Creating new region : " + hri);
937     HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
938     Path target = region.getRegionFileSystem().getRegionDir();
939 
940     // rename all the data to new region
941     mergeRegionDirs(target, hi);
942     fixes++;
943   }
944 
945   /**
946    * This method determines if there are table integrity errors in HDFS.  If
947    * there are errors and the appropriate "fix" options are enabled, the method
948    * will first correct orphan regions making them into legit regiondirs, and
949    * then reload to merge potentially overlapping regions.
950    *
951    * @return number of table integrity errors found
952    */
953   private int restoreHdfsIntegrity() throws IOException, InterruptedException {
954     // Determine what's on HDFS
955     LOG.info("Loading HBase regioninfo from HDFS...");
956     loadHdfsRegionDirs(); // populating regioninfo table.
957 
958     int errs = errors.getErrorList().size();
959     // First time just get suggestions.
960     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
961     checkHdfsIntegrity(false, false);
962 
963     if (errors.getErrorList().size() == errs) {
964       LOG.info("No integrity errors.  We are done with this phase. Glorious.");
965       return 0;
966     }
967 
968     if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
969       adoptHdfsOrphans(orphanHdfsDirs);
970       // TODO optimize by incrementally adding instead of reloading.
971     }
972 
973     // Make sure there are no holes now.
974     if (shouldFixHdfsHoles()) {
975       clearState(); // this also resets # fixes.
976       loadHdfsRegionDirs();
977       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
978       tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
979     }
980 
981     // Now we fix overlaps
982     if (shouldFixHdfsOverlaps()) {
983       // second pass we fix overlaps.
984       clearState(); // this also resets # fixes.
985       loadHdfsRegionDirs();
986       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
987       tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
988     }
989 
990     return errors.getErrorList().size();
991   }
992 
993   /**
994    * Scan all the store file names to find any lingering reference files,
995    * which refer to some none-exiting files. If "fix" option is enabled,
996    * any lingering reference file will be sidelined if found.
997    * <p>
998    * Lingering reference file prevents a region from opening. It has to
999    * be fixed before a cluster can start properly.
1000    */
1001   private void offlineReferenceFileRepair() throws IOException {
1002     Configuration conf = getConf();
1003     Path hbaseRoot = FSUtils.getRootDir(conf);
1004     FileSystem fs = hbaseRoot.getFileSystem(conf);
1005     LOG.info("Computing mapping of all store files");
1006     Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot, errors);
1007     errors.print("");
1008     LOG.info("Validating mapping using HDFS state");
1009     for (Path path: allFiles.values()) {
1010       boolean isReference = false;
1011       try {
1012         isReference = StoreFileInfo.isReference(path);
1013       } catch (Throwable t) {
1014         // Ignore. Some files may not be store files at all.
1015         // For example, files under .oldlogs folder in hbase:meta
1016         // Warning message is already logged by
1017         // StoreFile#isReference.
1018       }
1019       if (!isReference) continue;
1020 
1021       Path referredToFile = StoreFileInfo.getReferredToFile(path);
1022       if (fs.exists(referredToFile)) continue;  // good, expected
1023 
1024       // Found a lingering reference file
1025       errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
1026         "Found lingering reference file " + path);
1027       if (!shouldFixReferenceFiles()) continue;
1028 
1029       // Now, trying to fix it since requested
1030       boolean success = false;
1031       String pathStr = path.toString();
1032 
1033       // A reference file path should be like
1034       // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
1035       // Up 5 directories to get the root folder.
1036       // So the file will be sidelined to a similar folder structure.
1037       int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
1038       for (int i = 0; index > 0 && i < 5; i++) {
1039         index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
1040       }
1041       if (index > 0) {
1042         Path rootDir = getSidelineDir();
1043         Path dst = new Path(rootDir, pathStr.substring(index + 1));
1044         fs.mkdirs(dst.getParent());
1045         LOG.info("Trying to sildeline reference file "
1046           + path + " to " + dst);
1047         setShouldRerun();
1048 
1049         success = fs.rename(path, dst);
1050       }
1051       if (!success) {
1052         LOG.error("Failed to sideline reference file " + path);
1053       }
1054     }
1055   }
1056 
1057   /**
1058    * TODO -- need to add tests for this.
1059    */
1060   private void reportEmptyMetaCells() {
1061     errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
1062       emptyRegionInfoQualifiers.size());
1063     if (details) {
1064       for (Result r: emptyRegionInfoQualifiers) {
1065         errors.print("  " + r);
1066       }
1067     }
1068   }
1069 
1070   /**
1071    * TODO -- need to add tests for this.
1072    */
1073   private void reportTablesInFlux() {
1074     AtomicInteger numSkipped = new AtomicInteger(0);
1075     HTableDescriptor[] allTables = getTables(numSkipped);
1076     errors.print("Number of Tables: " + allTables.length);
1077     if (details) {
1078       if (numSkipped.get() > 0) {
1079         errors.detail("Number of Tables in flux: " + numSkipped.get());
1080       }
1081       for (HTableDescriptor td : allTables) {
1082         errors.detail("  Table: " + td.getTableName() + "\t" +
1083                            (td.isReadOnly() ? "ro" : "rw") + "\t" +
1084                             (td.isMetaRegion() ? "META" : "    ") + "\t" +
1085                            " families: " + td.getFamilies().size());
1086       }
1087     }
1088   }
1089 
1090   public ErrorReporter getErrors() {
1091     return errors;
1092   }
1093 
1094   /**
1095    * Read the .regioninfo file from the file system.  If there is no
1096    * .regioninfo, add it to the orphan hdfs region list.
1097    */
1098   private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
1099     Path regionDir = hbi.getHdfsRegionDir();
1100     if (regionDir == null) {
1101       LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
1102       return;
1103     }
1104 
1105     if (hbi.hdfsEntry.hri != null) {
1106       // already loaded data
1107       return;
1108     }
1109 
1110     FileSystem fs = FileSystem.get(getConf());
1111     HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
1112     LOG.debug("HRegionInfo read: " + hri.toString());
1113     hbi.hdfsEntry.hri = hri;
1114   }
1115 
1116   /**
1117    * Exception thrown when a integrity repair operation fails in an
1118    * unresolvable way.
1119    */
1120   public static class RegionRepairException extends IOException {
1121     private static final long serialVersionUID = 1L;
1122     final IOException ioe;
1123     public RegionRepairException(String s, IOException ioe) {
1124       super(s);
1125       this.ioe = ioe;
1126     }
1127   }
1128 
1129   /**
1130    * Populate hbi's from regionInfos loaded from file system.
1131    */
1132   private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
1133       throws IOException, InterruptedException {
1134     tablesInfo.clear(); // regenerating the data
1135     // generate region split structure
1136     Collection<HbckInfo> hbckInfos = regionInfoMap.values();
1137 
1138     // Parallelized read of .regioninfo files.
1139     List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
1140     List<Future<Void>> hbiFutures;
1141 
1142     for (HbckInfo hbi : hbckInfos) {
1143       WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1144       hbis.add(work);
1145     }
1146 
1147     // Submit and wait for completion
1148     hbiFutures = executor.invokeAll(hbis);
1149 
1150     for(int i=0; i<hbiFutures.size(); i++) {
1151       WorkItemHdfsRegionInfo work = hbis.get(i);
1152       Future<Void> f = hbiFutures.get(i);
1153       try {
1154         f.get();
1155       } catch(ExecutionException e) {
1156         LOG.warn("Failed to read .regioninfo file for region " +
1157               work.hbi.getRegionNameAsString(), e.getCause());
1158       }
1159     }
1160 
1161     Path hbaseRoot = FSUtils.getRootDir(getConf());
1162     FileSystem fs = hbaseRoot.getFileSystem(getConf());
1163     // serialized table info gathering.
1164     for (HbckInfo hbi: hbckInfos) {
1165 
1166       if (hbi.getHdfsHRI() == null) {
1167         // was an orphan
1168         continue;
1169       }
1170 
1171 
1172       // get table name from hdfs, populate various HBaseFsck tables.
1173       TableName tableName = hbi.getTableName();
1174       if (tableName == null) {
1175         // There was an entry in hbase:meta not in the HDFS?
1176         LOG.warn("tableName was null for: " + hbi);
1177         continue;
1178       }
1179 
1180       TableInfo modTInfo = tablesInfo.get(tableName);
1181       if (modTInfo == null) {
1182         // only executed once per table.
1183         modTInfo = new TableInfo(tableName);
1184         tablesInfo.put(tableName, modTInfo);
1185         try {
1186           TableDescriptor htd =
1187               FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1188           modTInfo.htds.add(htd.getHTableDescriptor());
1189         } catch (IOException ioe) {
1190           if (!orphanTableDirs.containsKey(tableName)) {
1191             LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1192             //should only report once for each table
1193             errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1194                 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1195             Set<String> columns = new HashSet<String>();
1196             orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1197           }
1198         }
1199       }
1200       if (!hbi.isSkipChecks()) {
1201         modTInfo.addRegionInfo(hbi);
1202       }
1203     }
1204 
1205     loadTableInfosForTablesWithNoRegion();
1206     errors.print("");
1207 
1208     return tablesInfo;
1209   }
1210 
1211   /**
1212    * To get the column family list according to the column family dirs
1213    * @param columns
1214    * @param hbi
1215    * @return a set of column families
1216    * @throws IOException
1217    */
1218   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
1219     Path regionDir = hbi.getHdfsRegionDir();
1220     FileSystem fs = regionDir.getFileSystem(getConf());
1221     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1222     for (FileStatus subdir : subDirs) {
1223       String columnfamily = subdir.getPath().getName();
1224       columns.add(columnfamily);
1225     }
1226     return columns;
1227   }
1228 
1229   /**
1230    * To fabricate a .tableinfo file with following contents<br>
1231    * 1. the correct tablename <br>
1232    * 2. the correct colfamily list<br>
1233    * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1234    * @throws IOException
1235    */
1236   private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1237       Set<String> columns) throws IOException {
1238     if (columns ==null || columns.isEmpty()) return false;
1239     HTableDescriptor htd = new HTableDescriptor(tableName);
1240     for (String columnfamimly : columns) {
1241       htd.addFamily(new HColumnDescriptor(columnfamimly));
1242     }
1243     fstd.createTableDescriptor(new TableDescriptor(htd), true);
1244     return true;
1245   }
1246 
1247   /**
1248    * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1249    * @throws IOException
1250    */
1251   public void fixEmptyMetaCells() throws IOException {
1252     if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1253       LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1254       for (Result region : emptyRegionInfoQualifiers) {
1255         deleteMetaRegion(region.getRow());
1256         errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1257       }
1258       emptyRegionInfoQualifiers.clear();
1259     }
1260   }
1261 
1262   /**
1263    * To fix orphan table by creating a .tableinfo file under tableDir <br>
1264    * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1265    * 2. else create a default .tableinfo file with following items<br>
1266    * &nbsp;2.1 the correct tablename <br>
1267    * &nbsp;2.2 the correct colfamily list<br>
1268    * &nbsp;2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1269    * @throws IOException
1270    */
1271   public void fixOrphanTables() throws IOException {
1272     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1273 
1274       List<TableName> tmpList = new ArrayList<TableName>();
1275       tmpList.addAll(orphanTableDirs.keySet());
1276       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1277       Iterator<Entry<TableName, Set<String>>> iter =
1278           orphanTableDirs.entrySet().iterator();
1279       int j = 0;
1280       int numFailedCase = 0;
1281       FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1282       while (iter.hasNext()) {
1283         Entry<TableName, Set<String>> entry =
1284             iter.next();
1285         TableName tableName = entry.getKey();
1286         LOG.info("Trying to fix orphan table error: " + tableName);
1287         if (j < htds.length) {
1288           if (tableName.equals(htds[j].getTableName())) {
1289             HTableDescriptor htd = htds[j];
1290             LOG.info("fixing orphan table: " + tableName + " from cache");
1291             fstd.createTableDescriptor(new TableDescriptor(htd), true);
1292             j++;
1293             iter.remove();
1294           }
1295         } else {
1296           if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1297             LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1298             LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1299             iter.remove();
1300           } else {
1301             LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1302             numFailedCase++;
1303           }
1304         }
1305         fixes++;
1306       }
1307 
1308       if (orphanTableDirs.isEmpty()) {
1309         // all orphanTableDirs are luckily recovered
1310         // re-run doFsck after recovering the .tableinfo file
1311         setShouldRerun();
1312         LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1313       } else if (numFailedCase > 0) {
1314         LOG.error("Failed to fix " + numFailedCase
1315             + " OrphanTables with default .tableinfo files");
1316       }
1317 
1318     }
1319     //cleanup the list
1320     orphanTableDirs.clear();
1321 
1322   }
1323 
1324   /**
1325    * This borrows code from MasterFileSystem.bootstrap(). Explicitly creates it's own WAL, so be
1326    * sure to close it as well as the region when you're finished.
1327    *
1328    * @return an open hbase:meta HRegion
1329    */
1330   private HRegion createNewMeta() throws IOException {
1331     Path rootdir = FSUtils.getRootDir(getConf());
1332     Configuration c = getConf();
1333     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1334     HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
1335     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false);
1336     // The WAL subsystem will use the default rootDir rather than the passed in rootDir
1337     // unless I pass along via the conf.
1338     Configuration confForWAL = new Configuration(c);
1339     confForWAL.set(HConstants.HBASE_DIR, rootdir.toString());
1340     WAL wal = (new WALFactory(confForWAL,
1341         Collections.<WALActionsListener>singletonList(new MetricsWAL()),
1342         "hbck-meta-recovery-" + RandomStringUtils.randomNumeric(8))).
1343         getWAL(metaHRI.getEncodedNameAsBytes());
1344     HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor, wal);
1345     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
1346     return meta;
1347   }
1348 
1349   /**
1350    * Generate set of puts to add to new meta.  This expects the tables to be
1351    * clean with no overlaps or holes.  If there are any problems it returns null.
1352    *
1353    * @return An array list of puts to do in bulk, null if tables have problems
1354    */
1355   private ArrayList<Put> generatePuts(
1356       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1357     ArrayList<Put> puts = new ArrayList<Put>();
1358     boolean hasProblems = false;
1359     for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1360       TableName name = e.getKey();
1361 
1362       // skip "hbase:meta"
1363       if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1364         continue;
1365       }
1366 
1367       TableInfo ti = e.getValue();
1368       puts.add(MetaTableAccessor
1369           .makePutFromTableState(new TableState(ti.tableName, TableState.State.ENABLED)));
1370       for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1371           .entrySet()) {
1372         Collection<HbckInfo> his = spl.getValue();
1373         int sz = his.size();
1374         if (sz != 1) {
1375           // problem
1376           LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1377               + " had " +  sz + " regions instead of exactly 1." );
1378           hasProblems = true;
1379           continue;
1380         }
1381 
1382         // add the row directly to meta.
1383         HbckInfo hi = his.iterator().next();
1384         HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
1385         Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1386         puts.add(p);
1387       }
1388     }
1389     return hasProblems ? null : puts;
1390   }
1391 
1392   /**
1393    * Suggest fixes for each table
1394    */
1395   private void suggestFixes(
1396       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1397     logParallelMerge();
1398     for (TableInfo tInfo : tablesInfo.values()) {
1399       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1400       tInfo.checkRegionChain(handler);
1401     }
1402   }
1403 
1404   /**
1405    * Rebuilds meta from information in hdfs/fs.  Depends on configuration settings passed into
1406    * hbck constructor to point to a particular fs/dir. Assumes HBase is OFFLINE.
1407    *
1408    * @param fix flag that determines if method should attempt to fix holes
1409    * @return true if successful, false if attempt failed.
1410    */
1411   public boolean rebuildMeta(boolean fix) throws IOException,
1412       InterruptedException {
1413 
1414     // TODO check to make sure hbase is offline. (or at least the table
1415     // currently being worked on is off line)
1416 
1417     // Determine what's on HDFS
1418     LOG.info("Loading HBase regioninfo from HDFS...");
1419     loadHdfsRegionDirs(); // populating regioninfo table.
1420 
1421     int errs = errors.getErrorList().size();
1422     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1423     checkHdfsIntegrity(false, false);
1424 
1425     // make sure ok.
1426     if (errors.getErrorList().size() != errs) {
1427       // While in error state, iterate until no more fixes possible
1428       while(true) {
1429         fixes = 0;
1430         suggestFixes(tablesInfo);
1431         errors.clear();
1432         loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1433         checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1434 
1435         int errCount = errors.getErrorList().size();
1436 
1437         if (fixes == 0) {
1438           if (errCount > 0) {
1439             return false; // failed to fix problems.
1440           } else {
1441             break; // no fixes and no problems? drop out and fix stuff!
1442           }
1443         }
1444       }
1445     }
1446 
1447     // we can rebuild, move old meta out of the way and start
1448     LOG.info("HDFS regioninfo's seems good.  Sidelining old hbase:meta");
1449     Path backupDir = sidelineOldMeta();
1450 
1451     LOG.info("Creating new hbase:meta");
1452     HRegion meta = createNewMeta();
1453 
1454     // populate meta
1455     List<Put> puts = generatePuts(tablesInfo);
1456     if (puts == null) {
1457       LOG.fatal("Problem encountered when creating new hbase:meta entries.  " +
1458         "You may need to restore the previously sidelined hbase:meta");
1459       return false;
1460     }
1461     meta.batchMutate(puts.toArray(new Put[puts.size()]));
1462     meta.close();
1463     if (meta.getWAL() != null) {
1464       meta.getWAL().close();
1465     }
1466     LOG.info("Success! hbase:meta table rebuilt.");
1467     LOG.info("Old hbase:meta is moved into " + backupDir);
1468     return true;
1469   }
1470 
1471   /**
1472    * Log an appropriate message about whether or not overlapping merges are computed in parallel.
1473    */
1474   private void logParallelMerge() {
1475     if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
1476       LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
1477           " false to run serially.");
1478     } else {
1479       LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
1480           " true to run in parallel.");
1481     }
1482   }
1483 
1484   private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1485       boolean fixOverlaps) throws IOException {
1486     LOG.info("Checking HBase region split map from HDFS data...");
1487     logParallelMerge();
1488     for (TableInfo tInfo : tablesInfo.values()) {
1489       TableIntegrityErrorHandler handler;
1490       if (fixHoles || fixOverlaps) {
1491         handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1492           fixHoles, fixOverlaps);
1493       } else {
1494         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1495       }
1496       if (!tInfo.checkRegionChain(handler)) {
1497         // should dump info as well.
1498         errors.report("Found inconsistency in table " + tInfo.getName());
1499       }
1500     }
1501     return tablesInfo;
1502   }
1503 
1504   private Path getSidelineDir() throws IOException {
1505     if (sidelineDir == null) {
1506       Path hbaseDir = FSUtils.getRootDir(getConf());
1507       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1508       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1509           + startMillis);
1510     }
1511     return sidelineDir;
1512   }
1513 
1514   /**
1515    * Sideline a region dir (instead of deleting it)
1516    */
1517   Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1518     return sidelineRegionDir(fs, null, hi);
1519   }
1520 
1521   /**
1522    * Sideline a region dir (instead of deleting it)
1523    *
1524    * @param parentDir if specified, the region will be sidelined to
1525    * folder like .../parentDir/<table name>/<region name>. The purpose
1526    * is to group together similar regions sidelined, for example, those
1527    * regions should be bulk loaded back later on. If null, it is ignored.
1528    */
1529   Path sidelineRegionDir(FileSystem fs,
1530       String parentDir, HbckInfo hi) throws IOException {
1531     TableName tableName = hi.getTableName();
1532     Path regionDir = hi.getHdfsRegionDir();
1533 
1534     if (!fs.exists(regionDir)) {
1535       LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1536       return null;
1537     }
1538 
1539     Path rootDir = getSidelineDir();
1540     if (parentDir != null) {
1541       rootDir = new Path(rootDir, parentDir);
1542     }
1543     Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1544     Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1545     fs.mkdirs(sidelineRegionDir);
1546     boolean success = false;
1547     FileStatus[] cfs =  fs.listStatus(regionDir);
1548     if (cfs == null) {
1549       LOG.info("Region dir is empty: " + regionDir);
1550     } else {
1551       for (FileStatus cf : cfs) {
1552         Path src = cf.getPath();
1553         Path dst =  new Path(sidelineRegionDir, src.getName());
1554         if (fs.isFile(src)) {
1555           // simple file
1556           success = fs.rename(src, dst);
1557           if (!success) {
1558             String msg = "Unable to rename file " + src +  " to " + dst;
1559             LOG.error(msg);
1560             throw new IOException(msg);
1561           }
1562           continue;
1563         }
1564 
1565         // is a directory.
1566         fs.mkdirs(dst);
1567 
1568         LOG.info("Sidelining files from " + src + " into containing region " + dst);
1569         // FileSystem.rename is inconsistent with directories -- if the
1570         // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1571         // it moves the src into the dst dir resulting in (foo/a/b).  If
1572         // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1573         FileStatus[] hfiles = fs.listStatus(src);
1574         if (hfiles != null && hfiles.length > 0) {
1575           for (FileStatus hfile : hfiles) {
1576             success = fs.rename(hfile.getPath(), dst);
1577             if (!success) {
1578               String msg = "Unable to rename file " + src +  " to " + dst;
1579               LOG.error(msg);
1580               throw new IOException(msg);
1581             }
1582           }
1583         }
1584         LOG.debug("Sideline directory contents:");
1585         debugLsr(sidelineRegionDir);
1586       }
1587     }
1588 
1589     LOG.info("Removing old region dir: " + regionDir);
1590     success = fs.delete(regionDir, true);
1591     if (!success) {
1592       String msg = "Unable to delete dir " + regionDir;
1593       LOG.error(msg);
1594       throw new IOException(msg);
1595     }
1596     return sidelineRegionDir;
1597   }
1598 
1599   /**
1600    * Side line an entire table.
1601    */
1602   void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1603       Path backupHbaseDir) throws IOException {
1604     Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1605     if (fs.exists(tableDir)) {
1606       Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1607       fs.mkdirs(backupTableDir.getParent());
1608       boolean success = fs.rename(tableDir, backupTableDir);
1609       if (!success) {
1610         throw new IOException("Failed to move  " + tableName + " from "
1611             +  tableDir + " to " + backupTableDir);
1612       }
1613     } else {
1614       LOG.info("No previous " + tableName +  " exists.  Continuing.");
1615     }
1616   }
1617 
1618   /**
1619    * @return Path to backup of original directory
1620    */
1621   Path sidelineOldMeta() throws IOException {
1622     // put current hbase:meta aside.
1623     Path hbaseDir = FSUtils.getRootDir(getConf());
1624     FileSystem fs = hbaseDir.getFileSystem(getConf());
1625     Path backupDir = getSidelineDir();
1626     fs.mkdirs(backupDir);
1627 
1628     try {
1629       sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1630     } catch (IOException e) {
1631         LOG.fatal("... failed to sideline meta. Currently in inconsistent state.  To restore "
1632             + "try to rename hbase:meta in " + backupDir.getName() + " to "
1633             + hbaseDir.getName() + ".", e);
1634       throw e; // throw original exception
1635     }
1636     return backupDir;
1637   }
1638 
1639   /**
1640    * Load the list of disabled tables in ZK into local set.
1641    * @throws ZooKeeperConnectionException
1642    * @throws IOException
1643    */
1644   private void loadTableStates()
1645   throws IOException {
1646     tableStates = MetaTableAccessor.getTableStates(connection);
1647   }
1648 
1649   /**
1650    * Check if the specified region's table is disabled.
1651    * @param tableName table to check status of
1652    */
1653   private boolean isTableDisabled(TableName tableName) {
1654     return tableStates.containsKey(tableName)
1655         && tableStates.get(tableName)
1656         .inStates(TableState.State.DISABLED, TableState.State.DISABLING);
1657   }
1658 
1659   /**
1660    * Scan HDFS for all regions, recording their information into
1661    * regionInfoMap
1662    */
1663   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1664     Path rootDir = FSUtils.getRootDir(getConf());
1665     FileSystem fs = rootDir.getFileSystem(getConf());
1666 
1667     // list all tables from HDFS
1668     List<FileStatus> tableDirs = Lists.newArrayList();
1669 
1670     boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1671 
1672     List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1673     for (Path path : paths) {
1674       TableName tableName = FSUtils.getTableName(path);
1675        if ((!checkMetaOnly &&
1676            isTableIncluded(tableName)) ||
1677            tableName.equals(TableName.META_TABLE_NAME)) {
1678          tableDirs.add(fs.getFileStatus(path));
1679        }
1680     }
1681 
1682     // verify that version file exists
1683     if (!foundVersionFile) {
1684       errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1685           "Version file does not exist in root dir " + rootDir);
1686       if (shouldFixVersionFile()) {
1687         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1688             + " file.");
1689         setShouldRerun();
1690         FSUtils.setVersion(fs, rootDir, getConf().getInt(
1691             HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1692             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1693             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1694       }
1695     }
1696 
1697     // level 1:  <HBASE_DIR>/*
1698     List<WorkItemHdfsDir> dirs = new ArrayList<WorkItemHdfsDir>(tableDirs.size());
1699     List<Future<Void>> dirsFutures;
1700 
1701     for (FileStatus tableDir : tableDirs) {
1702       LOG.debug("Loading region dirs from " +tableDir.getPath());
1703       dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir));
1704     }
1705 
1706     // Invoke and wait for Callables to complete
1707     dirsFutures = executor.invokeAll(dirs);
1708 
1709     for(Future<Void> f: dirsFutures) {
1710       try {
1711         f.get();
1712       } catch(ExecutionException e) {
1713         LOG.warn("Could not load region dir " , e.getCause());
1714       }
1715     }
1716     errors.print("");
1717   }
1718 
1719   /**
1720    * Record the location of the hbase:meta region as found in ZooKeeper.
1721    */
1722   private boolean recordMetaRegion() throws IOException {
1723     RegionLocations rl = ((ClusterConnection)connection).locateRegion(TableName.META_TABLE_NAME,
1724         HConstants.EMPTY_START_ROW, false, false);
1725     if (rl == null) {
1726       errors.reportError(ERROR_CODE.NULL_META_REGION,
1727           "META region was not found in Zookeeper");
1728       return false;
1729     }
1730     for (HRegionLocation metaLocation : rl.getRegionLocations()) {
1731       // Check if Meta region is valid and existing
1732       if (metaLocation == null ) {
1733         errors.reportError(ERROR_CODE.NULL_META_REGION,
1734             "META region location is null");
1735         return false;
1736       }
1737       if (metaLocation.getRegionInfo() == null) {
1738         errors.reportError(ERROR_CODE.NULL_META_REGION,
1739             "META location regionInfo is null");
1740         return false;
1741       }
1742       if (metaLocation.getHostname() == null) {
1743         errors.reportError(ERROR_CODE.NULL_META_REGION,
1744             "META location hostName is null");
1745         return false;
1746       }
1747       ServerName sn = metaLocation.getServerName();
1748       MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, EnvironmentEdgeManager.currentTime());
1749       HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1750       if (hbckInfo == null) {
1751         regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1752       } else {
1753         hbckInfo.metaEntry = m;
1754       }
1755     }
1756     return true;
1757   }
1758 
1759   private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1760     return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1761       @Override
1762       public void abort(String why, Throwable e) {
1763         LOG.error(why, e);
1764         System.exit(1);
1765       }
1766 
1767       @Override
1768       public boolean isAborted() {
1769         return false;
1770       }
1771 
1772     });
1773   }
1774 
1775   private ServerName getMetaRegionServerName(int replicaId)
1776   throws IOException, KeeperException {
1777     ZooKeeperWatcher zkw = createZooKeeperWatcher();
1778     ServerName sn = null;
1779     try {
1780       sn = new MetaTableLocator().getMetaRegionLocation(zkw, replicaId);
1781     } finally {
1782       zkw.close();
1783     }
1784     return sn;
1785   }
1786 
1787   /**
1788    * Contacts each regionserver and fetches metadata about regions.
1789    * @param regionServerList - the list of region servers to connect to
1790    * @throws IOException if a remote or network exception occurs
1791    */
1792   void processRegionServers(Collection<ServerName> regionServerList)
1793     throws IOException, InterruptedException {
1794 
1795     List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1796     List<Future<Void>> workFutures;
1797 
1798     // loop to contact each region server in parallel
1799     for (ServerName rsinfo: regionServerList) {
1800       workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1801     }
1802 
1803     workFutures = executor.invokeAll(workItems);
1804 
1805     for(int i=0; i<workFutures.size(); i++) {
1806       WorkItemRegion item = workItems.get(i);
1807       Future<Void> f = workFutures.get(i);
1808       try {
1809         f.get();
1810       } catch(ExecutionException e) {
1811         LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1812             e.getCause());
1813       }
1814     }
1815   }
1816 
1817   /**
1818    * Check consistency of all regions that have been found in previous phases.
1819    */
1820   private void checkAndFixConsistency()
1821   throws IOException, KeeperException, InterruptedException {
1822     // Divide the checks in two phases. One for default/primary replicas and another
1823     // for the non-primary ones. Keeps code cleaner this way.
1824 
1825     List<CheckRegionConsistencyWorkItem> workItems =
1826         new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1827     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1828       if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1829         workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1830       }
1831     }
1832     checkRegionConsistencyConcurrently(workItems);
1833 
1834     boolean prevHdfsCheck = shouldCheckHdfs();
1835     setCheckHdfs(false); //replicas don't have any hdfs data
1836     // Run a pass over the replicas and fix any assignment issues that exist on the currently
1837     // deployed/undeployed replicas.
1838     List<CheckRegionConsistencyWorkItem> replicaWorkItems =
1839         new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1840     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1841       if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1842         replicaWorkItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1843       }
1844     }
1845     checkRegionConsistencyConcurrently(replicaWorkItems);
1846     setCheckHdfs(prevHdfsCheck);
1847 
1848     // If some regions is skipped during checkRegionConsistencyConcurrently() phase, we might
1849     // not get accurate state of the hbase if continuing. The config here allows users to tune
1850     // the tolerance of number of skipped region.
1851     // TODO: evaluate the consequence to continue the hbck operation without config.
1852     int terminateThreshold =  getConf().getInt("hbase.hbck.skipped.regions.limit", 0);
1853     int numOfSkippedRegions = skippedRegions.size();
1854     if (numOfSkippedRegions > 0 && numOfSkippedRegions > terminateThreshold) {
1855       throw new IOException(numOfSkippedRegions
1856         + " region(s) could not be checked or repaired.  See logs for detail.");
1857     }
1858 
1859     if (shouldCheckHdfs()) {
1860       checkAndFixTableStates();
1861     }
1862   }
1863 
1864   /**
1865    * Check consistency of all regions using mulitple threads concurrently.
1866    */
1867   private void checkRegionConsistencyConcurrently(
1868     final List<CheckRegionConsistencyWorkItem> workItems)
1869     throws IOException, KeeperException, InterruptedException {
1870     if (workItems.isEmpty()) {
1871       return;  // nothing to check
1872     }
1873 
1874     List<Future<Void>> workFutures = executor.invokeAll(workItems);
1875     for(Future<Void> f: workFutures) {
1876       try {
1877         f.get();
1878       } catch(ExecutionException e1) {
1879         LOG.warn("Could not check region consistency " , e1.getCause());
1880         if (e1.getCause() instanceof IOException) {
1881           throw (IOException)e1.getCause();
1882         } else if (e1.getCause() instanceof KeeperException) {
1883           throw (KeeperException)e1.getCause();
1884         } else if (e1.getCause() instanceof InterruptedException) {
1885           throw (InterruptedException)e1.getCause();
1886         } else {
1887           throw new IOException(e1.getCause());
1888         }
1889       }
1890     }
1891   }
1892 
1893   class CheckRegionConsistencyWorkItem implements Callable<Void> {
1894     private final String key;
1895     private final HbckInfo hbi;
1896 
1897     CheckRegionConsistencyWorkItem(String key, HbckInfo hbi) {
1898       this.key = key;
1899       this.hbi = hbi;
1900     }
1901 
1902     @Override
1903     public synchronized Void call() throws Exception {
1904       try {
1905         checkRegionConsistency(key, hbi);
1906       } catch (Exception e) {
1907         // If the region is non-META region, skip this region and send warning/error message; if
1908         // the region is META region, we should not continue.
1909         LOG.warn("Unable to complete check or repair the region '" + hbi.getRegionNameAsString()
1910           + "'.", e);
1911         if (hbi.getHdfsHRI().isMetaRegion()) {
1912           throw e;
1913         }
1914         LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'");
1915         addSkippedRegion(hbi);
1916       }
1917       return null;
1918     }
1919   }
1920 
1921   private void addSkippedRegion(final HbckInfo hbi) {
1922     Set<String> skippedRegionNames = skippedRegions.get(hbi.getTableName());
1923     if (skippedRegionNames == null) {
1924       skippedRegionNames = new HashSet<String>();
1925     }
1926     skippedRegionNames.add(hbi.getRegionNameAsString());
1927     skippedRegions.put(hbi.getTableName(), skippedRegionNames);
1928   }
1929 
1930   /**
1931    * Check and fix table states, assumes full info available:
1932    * - tableInfos
1933    * - empty tables loaded
1934    */
1935   private void checkAndFixTableStates() throws IOException {
1936     // first check dangling states
1937     for (Entry<TableName, TableState> entry : tableStates.entrySet()) {
1938       TableName tableName = entry.getKey();
1939       TableState tableState = entry.getValue();
1940       TableInfo tableInfo = tablesInfo.get(tableName);
1941       if (isTableIncluded(tableName)
1942           && !tableName.isSystemTable()
1943           && tableInfo == null) {
1944         if (fixMeta) {
1945           MetaTableAccessor.deleteTableState(connection, tableName);
1946           TableState state = MetaTableAccessor.getTableState(connection, tableName);
1947           if (state != null) {
1948             errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,
1949                 tableName + " unable to delete dangling table state " + tableState);
1950           }
1951         } else {
1952           errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,
1953               tableName + " has dangling table state " + tableState);
1954         }
1955       }
1956     }
1957     // check that all tables have states
1958     for (TableName tableName : tablesInfo.keySet()) {
1959       if (isTableIncluded(tableName) && !tableStates.containsKey(tableName)) {
1960         if (fixMeta) {
1961           MetaTableAccessor.updateTableState(connection, tableName, TableState.State.ENABLED);
1962           TableState newState = MetaTableAccessor.getTableState(connection, tableName);
1963           if (newState == null) {
1964             errors.reportError(ERROR_CODE.NO_TABLE_STATE,
1965                 "Unable to change state for table " + tableName + " in meta ");
1966           }
1967         } else {
1968           errors.reportError(ERROR_CODE.NO_TABLE_STATE,
1969               tableName + " has no state in meta ");
1970         }
1971       }
1972     }
1973   }
1974 
1975   private void preCheckPermission() throws IOException, AccessDeniedException {
1976     if (shouldIgnorePreCheckPermission()) {
1977       return;
1978     }
1979 
1980     Path hbaseDir = FSUtils.getRootDir(getConf());
1981     FileSystem fs = hbaseDir.getFileSystem(getConf());
1982     UserProvider userProvider = UserProvider.instantiate(getConf());
1983     UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1984     FileStatus[] files = fs.listStatus(hbaseDir);
1985     for (FileStatus file : files) {
1986       try {
1987         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1988       } catch (AccessDeniedException ace) {
1989         LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1990         errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1991           + " does not have write perms to " + file.getPath()
1992           + ". Please rerun hbck as hdfs user " + file.getOwner());
1993         throw ace;
1994       }
1995     }
1996   }
1997 
1998   /**
1999    * Deletes region from meta table
2000    */
2001   private void deleteMetaRegion(HbckInfo hi) throws IOException {
2002     deleteMetaRegion(hi.metaEntry.getRegionName());
2003   }
2004 
2005   /**
2006    * Deletes region from meta table
2007    */
2008   private void deleteMetaRegion(byte[] metaKey) throws IOException {
2009     Delete d = new Delete(metaKey);
2010     meta.delete(d);
2011     LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
2012   }
2013 
2014   /**
2015    * Reset the split parent region info in meta table
2016    */
2017   private void resetSplitParent(HbckInfo hi) throws IOException {
2018     RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
2019     Delete d = new Delete(hi.metaEntry.getRegionName());
2020     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
2021     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
2022     mutations.add(d);
2023 
2024     HRegionInfo hri = new HRegionInfo(hi.metaEntry);
2025     hri.setOffline(false);
2026     hri.setSplit(false);
2027     Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
2028     mutations.add(p);
2029 
2030     meta.mutateRow(mutations);
2031     LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
2032   }
2033 
2034   /**
2035    * This backwards-compatibility wrapper for permanently offlining a region
2036    * that should not be alive.  If the region server does not support the
2037    * "offline" method, it will use the closest unassign method instead.  This
2038    * will basically work until one attempts to disable or delete the affected
2039    * table.  The problem has to do with in-memory only master state, so
2040    * restarting the HMaster or failing over to another should fix this.
2041    */
2042   private void offline(byte[] regionName) throws IOException {
2043     String regionString = Bytes.toStringBinary(regionName);
2044     if (!rsSupportsOffline) {
2045       LOG.warn("Using unassign region " + regionString
2046           + " instead of using offline method, you should"
2047           + " restart HMaster after these repairs");
2048       admin.unassign(regionName, true);
2049       return;
2050     }
2051 
2052     // first time we assume the rs's supports #offline.
2053     try {
2054       LOG.info("Offlining region " + regionString);
2055       admin.offline(regionName);
2056     } catch (IOException ioe) {
2057       String notFoundMsg = "java.lang.NoSuchMethodException: " +
2058         "org.apache.hadoop.hbase.master.HMaster.offline([B)";
2059       if (ioe.getMessage().contains(notFoundMsg)) {
2060         LOG.warn("Using unassign region " + regionString
2061             + " instead of using offline method, you should"
2062             + " restart HMaster after these repairs");
2063         rsSupportsOffline = false; // in the future just use unassign
2064         admin.unassign(regionName, true);
2065         return;
2066       }
2067       throw ioe;
2068     }
2069   }
2070 
2071   private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
2072     undeployRegionsForHbi(hi);
2073     // undeploy replicas of the region (but only if the method is invoked for the primary)
2074     if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2075       return;
2076     }
2077     int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2078     for (int i = 1; i < numReplicas; i++) {
2079       if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
2080       HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
2081           hi.getPrimaryHRIForDeployedReplica(), i);
2082       HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2083       if (h != null) {
2084         undeployRegionsForHbi(h);
2085         //set skip checks; we undeployed it, and we don't want to evaluate this anymore
2086         //in consistency checks
2087         h.setSkipChecks(true);
2088       }
2089     }
2090   }
2091 
2092   private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
2093     for (OnlineEntry rse : hi.deployedEntries) {
2094       LOG.debug("Undeploy region "  + rse.hri + " from " + rse.hsa);
2095       try {
2096         HBaseFsckRepair.closeRegionSilentlyAndWait(connection, rse.hsa, rse.hri);
2097         offline(rse.hri.getRegionName());
2098       } catch (IOException ioe) {
2099         LOG.warn("Got exception when attempting to offline region "
2100             + Bytes.toString(rse.hri.getRegionName()), ioe);
2101       }
2102     }
2103   }
2104 
2105   /**
2106    * Attempts to undeploy a region from a region server based in information in
2107    * META.  Any operations that modify the file system should make sure that
2108    * its corresponding region is not deployed to prevent data races.
2109    *
2110    * A separate call is required to update the master in-memory region state
2111    * kept in the AssignementManager.  Because disable uses this state instead of
2112    * that found in META, we can't seem to cleanly disable/delete tables that
2113    * have been hbck fixed.  When used on a version of HBase that does not have
2114    * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master
2115    * restart or failover may be required.
2116    */
2117   private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
2118     if (hi.metaEntry == null && hi.hdfsEntry == null) {
2119       undeployRegions(hi);
2120       return;
2121     }
2122 
2123     // get assignment info and hregioninfo from meta.
2124     Get get = new Get(hi.getRegionName());
2125     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2126     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
2127     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
2128     // also get the locations of the replicas to close if the primary region is being closed
2129     if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2130       int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2131       for (int i = 0; i < numReplicas; i++) {
2132         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
2133         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
2134       }
2135     }
2136     Result r = meta.get(get);
2137     RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
2138     if (rl == null) {
2139       LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
2140           " since meta does not have handle to reach it");
2141       return;
2142     }
2143     for (HRegionLocation h : rl.getRegionLocations()) {
2144       ServerName serverName = h.getServerName();
2145       if (serverName == null) {
2146         errors.reportError("Unable to close region "
2147             + hi.getRegionNameAsString() +  " because meta does not "
2148             + "have handle to reach it.");
2149         continue;
2150       }
2151       HRegionInfo hri = h.getRegionInfo();
2152       if (hri == null) {
2153         LOG.warn("Unable to close region " + hi.getRegionNameAsString()
2154             + " because hbase:meta had invalid or missing "
2155             + HConstants.CATALOG_FAMILY_STR + ":"
2156             + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
2157             + " qualifier value.");
2158         continue;
2159       }
2160       // close the region -- close files and remove assignment
2161       HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
2162     }
2163   }
2164 
2165   private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
2166     KeeperException, InterruptedException {
2167     // If we are trying to fix the errors
2168     if (shouldFixAssignments()) {
2169       errors.print(msg);
2170       undeployRegions(hbi);
2171       setShouldRerun();
2172       HRegionInfo hri = hbi.getHdfsHRI();
2173       if (hri == null) {
2174         hri = hbi.metaEntry;
2175       }
2176       HBaseFsckRepair.fixUnassigned(admin, hri);
2177       HBaseFsckRepair.waitUntilAssigned(admin, hri);
2178 
2179       // also assign replicas if needed (do it only when this call operates on a primary replica)
2180       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
2181       int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
2182       for (int i = 1; i < replicationCount; i++) {
2183         hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
2184         HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2185         if (h != null) {
2186           undeployRegions(h);
2187           //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore
2188           //in consistency checks
2189           h.setSkipChecks(true);
2190         }
2191         HBaseFsckRepair.fixUnassigned(admin, hri);
2192         HBaseFsckRepair.waitUntilAssigned(admin, hri);
2193       }
2194 
2195     }
2196   }
2197 
2198   /**
2199    * Check a single region for consistency and correct deployment.
2200    */
2201   private void checkRegionConsistency(final String key, final HbckInfo hbi)
2202   throws IOException, KeeperException, InterruptedException {
2203 
2204     if (hbi.isSkipChecks()) return;
2205     String descriptiveName = hbi.toString();
2206     boolean inMeta = hbi.metaEntry != null;
2207     // In case not checking HDFS, assume the region is on HDFS
2208     boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
2209     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
2210     boolean isDeployed = !hbi.deployedOn.isEmpty();
2211     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
2212     boolean deploymentMatchesMeta =
2213       hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
2214       hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
2215     boolean splitParent =
2216         inMeta && hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
2217     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry.getTable());
2218     boolean recentlyModified = inHdfs &&
2219       hbi.getModTime() + timelag > EnvironmentEdgeManager.currentTime();
2220 
2221     // ========== First the healthy cases =============
2222     if (hbi.containsOnlyHdfsEdits()) {
2223       return;
2224     }
2225     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
2226       return;
2227     } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
2228       LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
2229         "tabled that is not deployed");
2230       return;
2231     } else if (recentlyModified) {
2232       LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
2233       return;
2234     }
2235     // ========== Cases where the region is not in hbase:meta =============
2236     else if (!inMeta && !inHdfs && !isDeployed) {
2237       // We shouldn't have record of this region at all then!
2238       assert false : "Entry for region with no data";
2239     } else if (!inMeta && !inHdfs && isDeployed) {
2240       errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
2241           + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
2242           "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2243       if (shouldFixAssignments()) {
2244         undeployRegions(hbi);
2245       }
2246 
2247     } else if (!inMeta && inHdfs && !isDeployed) {
2248       if (hbi.isMerged()) {
2249         // This region has already been merged, the remaining hdfs file will be
2250         // cleaned by CatalogJanitor later
2251         hbi.setSkipChecks(true);
2252         LOG.info("Region " + descriptiveName
2253             + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
2254         return;
2255       }
2256       errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
2257           + descriptiveName + " on HDFS, but not listed in hbase:meta " +
2258           "or deployed on any region server");
2259       // restore region consistency of an adopted orphan
2260       if (shouldFixMeta()) {
2261         if (!hbi.isHdfsRegioninfoPresent()) {
2262           LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
2263               +  " in table integrity repair phase if -fixHdfsOrphans was" +
2264               " used.");
2265           return;
2266         }
2267 
2268         HRegionInfo hri = hbi.getHdfsHRI();
2269         TableInfo tableInfo = tablesInfo.get(hri.getTable());
2270 
2271         for (HRegionInfo region : tableInfo.getRegionsFromMeta()) {
2272           if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0
2273               && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(),
2274                 hri.getEndKey()) >= 0)
2275               && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) {
2276             if(region.isSplit() || region.isOffline()) continue;
2277             Path regionDir = hbi.getHdfsRegionDir();
2278             FileSystem fs = regionDir.getFileSystem(getConf());
2279             List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir);
2280             for (Path familyDir : familyDirs) {
2281               List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);
2282               for (Path referenceFilePath : referenceFilePaths) {
2283                 Path parentRegionDir =
2284                     StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();
2285                 if (parentRegionDir.toString().endsWith(region.getEncodedName())) {
2286                   LOG.warn(hri + " start and stop keys are in the range of " + region
2287                       + ". The region might not be cleaned up from hdfs when region " + region
2288                       + " split failed. Hence deleting from hdfs.");
2289                   HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
2290                     regionDir.getParent(), hri);
2291                   return;
2292                 }
2293               }
2294             }
2295           }
2296         }
2297         LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
2298         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2299         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2300             admin.getClusterStatus().getServers(), numReplicas);
2301 
2302         tryAssignmentRepair(hbi, "Trying to reassign region...");
2303       }
2304 
2305     } else if (!inMeta && inHdfs && isDeployed) {
2306       errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
2307           + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2308       debugLsr(hbi.getHdfsRegionDir());
2309       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2310         // for replicas, this means that we should undeploy the region (we would have
2311         // gone over the primaries and fixed meta holes in first phase under
2312         // checkAndFixConsistency; we shouldn't get the condition !inMeta at
2313         // this stage unless unwanted replica)
2314         if (shouldFixAssignments()) {
2315           undeployRegionsForHbi(hbi);
2316         }
2317       }
2318       if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2319         if (!hbi.isHdfsRegioninfoPresent()) {
2320           LOG.error("This should have been repaired in table integrity repair phase");
2321           return;
2322         }
2323 
2324         LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
2325         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2326         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2327             admin.getClusterStatus().getServers(), numReplicas);
2328         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2329       }
2330 
2331     // ========== Cases where the region is in hbase:meta =============
2332     } else if (inMeta && inHdfs && !isDeployed && splitParent) {
2333       // check whether this is an actual error, or just transient state where parent
2334       // is not cleaned
2335       if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
2336         // check that split daughters are there
2337         HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
2338         HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
2339         if (infoA != null && infoB != null) {
2340           // we already processed or will process daughters. Move on, nothing to see here.
2341           hbi.setSkipChecks(true);
2342           return;
2343         }
2344       }
2345       errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2346           + descriptiveName + " is a split parent in META, in HDFS, "
2347           + "and not deployed on any region server. This could be transient.");
2348       if (shouldFixSplitParents()) {
2349         setShouldRerun();
2350         resetSplitParent(hbi);
2351       }
2352     } else if (inMeta && !inHdfs && !isDeployed) {
2353       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2354           + descriptiveName + " found in META, but not in HDFS "
2355           + "or deployed on any region server.");
2356       if (shouldFixMeta()) {
2357         deleteMetaRegion(hbi);
2358       }
2359     } else if (inMeta && !inHdfs && isDeployed) {
2360       errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2361           + " found in META, but not in HDFS, " +
2362           "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2363       // We treat HDFS as ground truth.  Any information in meta is transient
2364       // and equivalent data can be regenerated.  So, lets unassign and remove
2365       // these problems from META.
2366       if (shouldFixAssignments()) {
2367         errors.print("Trying to fix unassigned region...");
2368         undeployRegions(hbi);
2369       }
2370       if (shouldFixMeta()) {
2371         // wait for it to complete
2372         deleteMetaRegion(hbi);
2373       }
2374     } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2375       errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2376           + " not deployed on any region server.");
2377       tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2378     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2379       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2380           "Region " + descriptiveName + " should not be deployed according " +
2381           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2382       if (shouldFixAssignments()) {
2383         errors.print("Trying to close the region " + descriptiveName);
2384         setShouldRerun();
2385         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2386       }
2387     } else if (inMeta && inHdfs && isMultiplyDeployed) {
2388       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2389           + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
2390           + " but is multiply assigned to region servers " +
2391           Joiner.on(", ").join(hbi.deployedOn));
2392       // If we are trying to fix the errors
2393       if (shouldFixAssignments()) {
2394         errors.print("Trying to fix assignment error...");
2395         setShouldRerun();
2396         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2397       }
2398     } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2399       errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2400           + descriptiveName + " listed in hbase:meta on region server " +
2401           hbi.metaEntry.regionServer + " but found on region server " +
2402           hbi.deployedOn.get(0));
2403       // If we are trying to fix the errors
2404       if (shouldFixAssignments()) {
2405         errors.print("Trying to fix assignment error...");
2406         setShouldRerun();
2407         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2408         HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2409       }
2410     } else {
2411       errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2412           " is in an unforeseen state:" +
2413           " inMeta=" + inMeta +
2414           " inHdfs=" + inHdfs +
2415           " isDeployed=" + isDeployed +
2416           " isMultiplyDeployed=" + isMultiplyDeployed +
2417           " deploymentMatchesMeta=" + deploymentMatchesMeta +
2418           " shouldBeDeployed=" + shouldBeDeployed);
2419     }
2420   }
2421 
2422   /**
2423    * Checks tables integrity. Goes over all regions and scans the tables.
2424    * Collects all the pieces for each table and checks if there are missing,
2425    * repeated or overlapping ones.
2426    * @throws IOException
2427    */
2428   SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2429     tablesInfo = new TreeMap<TableName,TableInfo> ();
2430     LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2431     for (HbckInfo hbi : regionInfoMap.values()) {
2432       // Check only valid, working regions
2433       if (hbi.metaEntry == null) {
2434         // this assumes that consistency check has run loadMetaEntry
2435         Path p = hbi.getHdfsRegionDir();
2436         if (p == null) {
2437           errors.report("No regioninfo in Meta or HDFS. " + hbi);
2438         }
2439 
2440         // TODO test.
2441         continue;
2442       }
2443       if (hbi.metaEntry.regionServer == null) {
2444         errors.detail("Skipping region because no region server: " + hbi);
2445         continue;
2446       }
2447       if (hbi.metaEntry.isOffline()) {
2448         errors.detail("Skipping region because it is offline: " + hbi);
2449         continue;
2450       }
2451       if (hbi.containsOnlyHdfsEdits()) {
2452         errors.detail("Skipping region because it only contains edits" + hbi);
2453         continue;
2454       }
2455 
2456       // Missing regionDir or over-deployment is checked elsewhere. Include
2457       // these cases in modTInfo, so we can evaluate those regions as part of
2458       // the region chain in META
2459       //if (hbi.foundRegionDir == null) continue;
2460       //if (hbi.deployedOn.size() != 1) continue;
2461       if (hbi.deployedOn.size() == 0) continue;
2462 
2463       // We should be safe here
2464       TableName tableName = hbi.metaEntry.getTable();
2465       TableInfo modTInfo = tablesInfo.get(tableName);
2466       if (modTInfo == null) {
2467         modTInfo = new TableInfo(tableName);
2468       }
2469       for (ServerName server : hbi.deployedOn) {
2470         modTInfo.addServer(server);
2471       }
2472 
2473       if (!hbi.isSkipChecks()) {
2474         modTInfo.addRegionInfo(hbi);
2475       }
2476 
2477       tablesInfo.put(tableName, modTInfo);
2478     }
2479 
2480     loadTableInfosForTablesWithNoRegion();
2481 
2482     logParallelMerge();
2483     for (TableInfo tInfo : tablesInfo.values()) {
2484       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2485       if (!tInfo.checkRegionChain(handler)) {
2486         errors.report("Found inconsistency in table " + tInfo.getName());
2487       }
2488     }
2489     return tablesInfo;
2490   }
2491 
2492   /** Loads table info's for tables that may not have been included, since there are no
2493    * regions reported for the table, but table dir is there in hdfs
2494    */
2495   private void loadTableInfosForTablesWithNoRegion() throws IOException {
2496     Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2497     for (HTableDescriptor htd : allTables.values()) {
2498       if (checkMetaOnly && !htd.isMetaTable()) {
2499         continue;
2500       }
2501 
2502       TableName tableName = htd.getTableName();
2503       if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2504         TableInfo tableInfo = new TableInfo(tableName);
2505         tableInfo.htds.add(htd);
2506         tablesInfo.put(htd.getTableName(), tableInfo);
2507       }
2508     }
2509   }
2510 
2511   /**
2512    * Merge hdfs data by moving from contained HbckInfo into targetRegionDir.
2513    * @return number of file move fixes done to merge regions.
2514    */
2515   public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2516     int fileMoves = 0;
2517     String thread = Thread.currentThread().getName();
2518     LOG.debug("[" + thread + "] Contained region dir after close and pause");
2519     debugLsr(contained.getHdfsRegionDir());
2520 
2521     // rename the contained into the container.
2522     FileSystem fs = targetRegionDir.getFileSystem(getConf());
2523     FileStatus[] dirs = null;
2524     try {
2525       dirs = fs.listStatus(contained.getHdfsRegionDir());
2526     } catch (FileNotFoundException fnfe) {
2527       // region we are attempting to merge in is not present!  Since this is a merge, there is
2528       // no harm skipping this region if it does not exist.
2529       if (!fs.exists(contained.getHdfsRegionDir())) {
2530         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2531             + " is missing. Assuming already sidelined or moved.");
2532       } else {
2533         sidelineRegionDir(fs, contained);
2534       }
2535       return fileMoves;
2536     }
2537 
2538     if (dirs == null) {
2539       if (!fs.exists(contained.getHdfsRegionDir())) {
2540         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2541             + " already sidelined.");
2542       } else {
2543         sidelineRegionDir(fs, contained);
2544       }
2545       return fileMoves;
2546     }
2547 
2548     for (FileStatus cf : dirs) {
2549       Path src = cf.getPath();
2550       Path dst =  new Path(targetRegionDir, src.getName());
2551 
2552       if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2553         // do not copy the old .regioninfo file.
2554         continue;
2555       }
2556 
2557       if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2558         // do not copy the .oldlogs files
2559         continue;
2560       }
2561 
2562       LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2563       // FileSystem.rename is inconsistent with directories -- if the
2564       // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2565       // it moves the src into the dst dir resulting in (foo/a/b).  If
2566       // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2567       for (FileStatus hfile : fs.listStatus(src)) {
2568         boolean success = fs.rename(hfile.getPath(), dst);
2569         if (success) {
2570           fileMoves++;
2571         }
2572       }
2573       LOG.debug("[" + thread + "] Sideline directory contents:");
2574       debugLsr(targetRegionDir);
2575     }
2576 
2577     // if all success.
2578     sidelineRegionDir(fs, contained);
2579     LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2580         getSidelineDir());
2581     debugLsr(contained.getHdfsRegionDir());
2582 
2583     return fileMoves;
2584   }
2585 
2586 
2587   static class WorkItemOverlapMerge implements Callable<Void> {
2588     private TableIntegrityErrorHandler handler;
2589     Collection<HbckInfo> overlapgroup;
2590 
2591     WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2592       this.handler = handler;
2593       this.overlapgroup = overlapgroup;
2594     }
2595 
2596     @Override
2597     public Void call() throws Exception {
2598       handler.handleOverlapGroup(overlapgroup);
2599       return null;
2600     }
2601   };
2602 
2603 
2604   /**
2605    * Maintain information about a particular table.
2606    */
2607   public class TableInfo {
2608     TableName tableName;
2609     TreeSet <ServerName> deployedOn;
2610 
2611     // backwards regions
2612     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2613 
2614     // sidelined big overlapped regions
2615     final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2616 
2617     // region split calculator
2618     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2619 
2620     // Histogram of different HTableDescriptors found.  Ideally there is only one!
2621     final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2622 
2623     // key = start split, values = set of splits in problem group
2624     final Multimap<byte[], HbckInfo> overlapGroups =
2625       TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2626 
2627     // list of regions derived from meta entries.
2628     private ImmutableList<HRegionInfo> regionsFromMeta = null;
2629 
2630     TableInfo(TableName name) {
2631       this.tableName = name;
2632       deployedOn = new TreeSet <ServerName>();
2633     }
2634 
2635     /**
2636      * @return descriptor common to all regions.  null if are none or multiple!
2637      */
2638     private HTableDescriptor getHTD() {
2639       if (htds.size() == 1) {
2640         return (HTableDescriptor)htds.toArray()[0];
2641       } else {
2642         LOG.error("None/Multiple table descriptors found for table '"
2643           + tableName + "' regions: " + htds);
2644       }
2645       return null;
2646     }
2647 
2648     public void addRegionInfo(HbckInfo hir) {
2649       if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2650         // end key is absolute end key, just add it.
2651         // ignore replicas other than primary for these checks
2652         if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2653         return;
2654       }
2655 
2656       // if not the absolute end key, check for cycle
2657       if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2658         errors.reportError(
2659             ERROR_CODE.REGION_CYCLE,
2660             String.format("The endkey for this region comes before the "
2661                 + "startkey, startkey=%s, endkey=%s",
2662                 Bytes.toStringBinary(hir.getStartKey()),
2663                 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2664         backwards.add(hir);
2665         return;
2666       }
2667 
2668       // main case, add to split calculator
2669       // ignore replicas other than primary for these checks
2670       if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2671     }
2672 
2673     public void addServer(ServerName server) {
2674       this.deployedOn.add(server);
2675     }
2676 
2677     public TableName getName() {
2678       return tableName;
2679     }
2680 
2681     public int getNumRegions() {
2682       return sc.getStarts().size() + backwards.size();
2683     }
2684 
2685     public synchronized ImmutableList<HRegionInfo> getRegionsFromMeta() {
2686       // lazy loaded, synchronized to ensure a single load
2687       if (regionsFromMeta == null) {
2688         List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
2689         for (HbckInfo h : HBaseFsck.this.regionInfoMap.values()) {
2690           if (tableName.equals(h.getTableName())) {
2691             if (h.metaEntry != null) {
2692               regions.add((HRegionInfo) h.metaEntry);
2693             }
2694           }
2695         }
2696         regionsFromMeta = Ordering.natural().immutableSortedCopy(regions);
2697       }
2698 
2699       return regionsFromMeta;
2700     }
2701 
2702     private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2703       ErrorReporter errors;
2704 
2705       IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2706         this.errors = errors;
2707         setTableInfo(ti);
2708       }
2709 
2710       @Override
2711       public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2712         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2713             "First region should start with an empty key.  You need to "
2714             + " create a new region and regioninfo in HDFS to plug the hole.",
2715             getTableInfo(), hi);
2716       }
2717 
2718       @Override
2719       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2720         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2721             "Last region should end with an empty key. You need to "
2722                 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2723       }
2724 
2725       @Override
2726       public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2727         errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2728             "Region has the same start and end key.", getTableInfo(), hi);
2729       }
2730 
2731       @Override
2732       public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2733         byte[] key = r1.getStartKey();
2734         // dup start key
2735         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2736             "Multiple regions have the same startkey: "
2737             + Bytes.toStringBinary(key), getTableInfo(), r1);
2738         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2739             "Multiple regions have the same startkey: "
2740             + Bytes.toStringBinary(key), getTableInfo(), r2);
2741       }
2742 
2743       @Override
2744       public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2745         errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2746             "There is an overlap in the region chain.",
2747             getTableInfo(), hi1, hi2);
2748       }
2749 
2750       @Override
2751       public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2752         errors.reportError(
2753             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2754             "There is a hole in the region chain between "
2755                 + Bytes.toStringBinary(holeStart) + " and "
2756                 + Bytes.toStringBinary(holeStop)
2757                 + ".  You need to create a new .regioninfo and region "
2758                 + "dir in hdfs to plug the hole.");
2759       }
2760     };
2761 
2762     /**
2763      * This handler fixes integrity errors from hdfs information.  There are
2764      * basically three classes of integrity problems 1) holes, 2) overlaps, and
2765      * 3) invalid regions.
2766      *
2767      * This class overrides methods that fix holes and the overlap group case.
2768      * Individual cases of particular overlaps are handled by the general
2769      * overlap group merge repair case.
2770      *
2771      * If hbase is online, this forces regions offline before doing merge
2772      * operations.
2773      */
2774     private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2775       Configuration conf;
2776 
2777       boolean fixOverlaps = true;
2778 
2779       HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2780           boolean fixHoles, boolean fixOverlaps) {
2781         super(ti, errors);
2782         this.conf = conf;
2783         this.fixOverlaps = fixOverlaps;
2784         // TODO properly use fixHoles
2785       }
2786 
2787       /**
2788        * This is a special case hole -- when the first region of a table is
2789        * missing from META, HBase doesn't acknowledge the existance of the
2790        * table.
2791        */
2792       @Override
2793       public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2794         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2795             "First region should start with an empty key.  Creating a new " +
2796             "region and regioninfo in HDFS to plug the hole.",
2797             getTableInfo(), next);
2798         HTableDescriptor htd = getTableInfo().getHTD();
2799         // from special EMPTY_START_ROW to next region's startKey
2800         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2801             HConstants.EMPTY_START_ROW, next.getStartKey());
2802 
2803         // TODO test
2804         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2805         LOG.info("Table region start key was not empty.  Created new empty region: "
2806             + newRegion + " " +region);
2807         fixes++;
2808       }
2809 
2810       @Override
2811       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2812         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2813             "Last region should end with an empty key.  Creating a new "
2814                 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2815         HTableDescriptor htd = getTableInfo().getHTD();
2816         // from curEndKey to EMPTY_START_ROW
2817         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2818             HConstants.EMPTY_START_ROW);
2819 
2820         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2821         LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
2822             + " " + region);
2823         fixes++;
2824       }
2825 
2826       /**
2827        * There is a hole in the hdfs regions that violates the table integrity
2828        * rules.  Create a new empty region that patches the hole.
2829        */
2830       @Override
2831       public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2832         errors.reportError(
2833             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2834             "There is a hole in the region chain between "
2835                 + Bytes.toStringBinary(holeStartKey) + " and "
2836                 + Bytes.toStringBinary(holeStopKey)
2837                 + ".  Creating a new regioninfo and region "
2838                 + "dir in hdfs to plug the hole.");
2839         HTableDescriptor htd = getTableInfo().getHTD();
2840         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2841         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2842         LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2843         fixes++;
2844       }
2845 
2846       /**
2847        * This takes set of overlapping regions and merges them into a single
2848        * region.  This covers cases like degenerate regions, shared start key,
2849        * general overlaps, duplicate ranges, and partial overlapping regions.
2850        *
2851        * Cases:
2852        * - Clean regions that overlap
2853        * - Only .oldlogs regions (can't find start/stop range, or figure out)
2854        *
2855        * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
2856        */
2857       @Override
2858       public void handleOverlapGroup(Collection<HbckInfo> overlap)
2859           throws IOException {
2860         Preconditions.checkNotNull(overlap);
2861         Preconditions.checkArgument(overlap.size() >0);
2862 
2863         if (!this.fixOverlaps) {
2864           LOG.warn("Not attempting to repair overlaps.");
2865           return;
2866         }
2867 
2868         if (overlap.size() > maxMerge) {
2869           LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2870             "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2871           if (sidelineBigOverlaps) {
2872             // we only sideline big overlapped groups that exceeds the max number of regions to merge
2873             sidelineBigOverlaps(overlap);
2874           }
2875           return;
2876         }
2877 
2878         mergeOverlaps(overlap);
2879       }
2880 
2881       void mergeOverlaps(Collection<HbckInfo> overlap)
2882           throws IOException {
2883         String thread = Thread.currentThread().getName();
2884         LOG.info("== [" + thread + "] Merging regions into one region: "
2885           + Joiner.on(",").join(overlap));
2886         // get the min / max range and close all concerned regions
2887         Pair<byte[], byte[]> range = null;
2888         for (HbckInfo hi : overlap) {
2889           if (range == null) {
2890             range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2891           } else {
2892             if (RegionSplitCalculator.BYTES_COMPARATOR
2893                 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2894               range.setFirst(hi.getStartKey());
2895             }
2896             if (RegionSplitCalculator.BYTES_COMPARATOR
2897                 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2898               range.setSecond(hi.getEndKey());
2899             }
2900           }
2901           // need to close files so delete can happen.
2902           LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
2903           LOG.debug("[" + thread + "] Contained region dir before close");
2904           debugLsr(hi.getHdfsRegionDir());
2905           try {
2906             LOG.info("[" + thread + "] Closing region: " + hi);
2907             closeRegion(hi);
2908           } catch (IOException ioe) {
2909             LOG.warn("[" + thread + "] Was unable to close region " + hi
2910               + ".  Just continuing... ", ioe);
2911           } catch (InterruptedException e) {
2912             LOG.warn("[" + thread + "] Was unable to close region " + hi
2913               + ".  Just continuing... ", e);
2914           }
2915 
2916           try {
2917             LOG.info("[" + thread + "] Offlining region: " + hi);
2918             offline(hi.getRegionName());
2919           } catch (IOException ioe) {
2920             LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2921               + ".  Just continuing... ", ioe);
2922           }
2923         }
2924 
2925         // create new empty container region.
2926         HTableDescriptor htd = getTableInfo().getHTD();
2927         // from start key to end Key
2928         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
2929             range.getSecond());
2930         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2931         LOG.info("[" + thread + "] Created new empty container region: " +
2932             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
2933         debugLsr(region.getRegionFileSystem().getRegionDir());
2934 
2935         // all target regions are closed, should be able to safely cleanup.
2936         boolean didFix= false;
2937         Path target = region.getRegionFileSystem().getRegionDir();
2938         for (HbckInfo contained : overlap) {
2939           LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
2940           int merges = mergeRegionDirs(target, contained);
2941           if (merges > 0) {
2942             didFix = true;
2943           }
2944         }
2945         if (didFix) {
2946           fixes++;
2947         }
2948       }
2949 
2950       /**
2951        * Sideline some regions in a big overlap group so that it
2952        * will have fewer regions, and it is easier to merge them later on.
2953        *
2954        * @param bigOverlap the overlapped group with regions more than maxMerge
2955        * @throws IOException
2956        */
2957       void sidelineBigOverlaps(
2958           Collection<HbckInfo> bigOverlap) throws IOException {
2959         int overlapsToSideline = bigOverlap.size() - maxMerge;
2960         if (overlapsToSideline > maxOverlapsToSideline) {
2961           overlapsToSideline = maxOverlapsToSideline;
2962         }
2963         List<HbckInfo> regionsToSideline =
2964           RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
2965         FileSystem fs = FileSystem.get(conf);
2966         for (HbckInfo regionToSideline: regionsToSideline) {
2967           try {
2968             LOG.info("Closing region: " + regionToSideline);
2969             closeRegion(regionToSideline);
2970           } catch (IOException ioe) {
2971             LOG.warn("Was unable to close region " + regionToSideline
2972               + ".  Just continuing... ", ioe);
2973           } catch (InterruptedException e) {
2974             LOG.warn("Was unable to close region " + regionToSideline
2975               + ".  Just continuing... ", e);
2976           }
2977 
2978           try {
2979             LOG.info("Offlining region: " + regionToSideline);
2980             offline(regionToSideline.getRegionName());
2981           } catch (IOException ioe) {
2982             LOG.warn("Unable to offline region from master: " + regionToSideline
2983               + ".  Just continuing... ", ioe);
2984           }
2985 
2986           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
2987           Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
2988           if (sidelineRegionDir != null) {
2989             sidelinedRegions.put(sidelineRegionDir, regionToSideline);
2990             LOG.info("After sidelined big overlapped region: "
2991               + regionToSideline.getRegionNameAsString()
2992               + " to " + sidelineRegionDir.toString());
2993             fixes++;
2994           }
2995         }
2996       }
2997     }
2998 
2999     /**
3000      * Check the region chain (from META) of this table.  We are looking for
3001      * holes, overlaps, and cycles.
3002      * @return false if there are errors
3003      * @throws IOException
3004      */
3005     public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
3006       // When table is disabled no need to check for the region chain. Some of the regions
3007       // accidently if deployed, this below code might report some issues like missing start
3008       // or end regions or region hole in chain and may try to fix which is unwanted.
3009       if (isTableDisabled(this.tableName)) {
3010         return true;
3011       }
3012       int originalErrorsCount = errors.getErrorList().size();
3013       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
3014       SortedSet<byte[]> splits = sc.getSplits();
3015 
3016       byte[] prevKey = null;
3017       byte[] problemKey = null;
3018 
3019       if (splits.size() == 0) {
3020         // no region for this table
3021         handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
3022       }
3023 
3024       for (byte[] key : splits) {
3025         Collection<HbckInfo> ranges = regions.get(key);
3026         if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
3027           for (HbckInfo rng : ranges) {
3028             handler.handleRegionStartKeyNotEmpty(rng);
3029           }
3030         }
3031 
3032         // check for degenerate ranges
3033         for (HbckInfo rng : ranges) {
3034           // special endkey case converts '' to null
3035           byte[] endKey = rng.getEndKey();
3036           endKey = (endKey.length == 0) ? null : endKey;
3037           if (Bytes.equals(rng.getStartKey(),endKey)) {
3038             handler.handleDegenerateRegion(rng);
3039           }
3040         }
3041 
3042         if (ranges.size() == 1) {
3043           // this split key is ok -- no overlap, not a hole.
3044           if (problemKey != null) {
3045             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3046           }
3047           problemKey = null; // fell through, no more problem.
3048         } else if (ranges.size() > 1) {
3049           // set the new problem key group name, if already have problem key, just
3050           // keep using it.
3051           if (problemKey == null) {
3052             // only for overlap regions.
3053             LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
3054             problemKey = key;
3055           }
3056           overlapGroups.putAll(problemKey, ranges);
3057 
3058           // record errors
3059           ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
3060           //  this dumb and n^2 but this shouldn't happen often
3061           for (HbckInfo r1 : ranges) {
3062             if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3063             subRange.remove(r1);
3064             for (HbckInfo r2 : subRange) {
3065               if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3066               if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
3067                 handler.handleDuplicateStartKeys(r1,r2);
3068               } else {
3069                 // overlap
3070                 handler.handleOverlapInRegionChain(r1, r2);
3071               }
3072             }
3073           }
3074 
3075         } else if (ranges.size() == 0) {
3076           if (problemKey != null) {
3077             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3078           }
3079           problemKey = null;
3080 
3081           byte[] holeStopKey = sc.getSplits().higher(key);
3082           // if higher key is null we reached the top.
3083           if (holeStopKey != null) {
3084             // hole
3085             handler.handleHoleInRegionChain(key, holeStopKey);
3086           }
3087         }
3088         prevKey = key;
3089       }
3090 
3091       // When the last region of a table is proper and having an empty end key, 'prevKey'
3092       // will be null.
3093       if (prevKey != null) {
3094         handler.handleRegionEndKeyNotEmpty(prevKey);
3095       }
3096 
3097       // TODO fold this into the TableIntegrityHandler
3098       if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
3099         boolean ok = handleOverlapsParallel(handler, prevKey);
3100         if (!ok) {
3101           return false;
3102         }
3103       } else {
3104         for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3105           handler.handleOverlapGroup(overlap);
3106         }
3107       }
3108 
3109       if (details) {
3110         // do full region split map dump
3111         errors.print("---- Table '"  +  this.tableName
3112             + "': region split map");
3113         dump(splits, regions);
3114         errors.print("---- Table '"  +  this.tableName
3115             + "': overlap groups");
3116         dumpOverlapProblems(overlapGroups);
3117         errors.print("There are " + overlapGroups.keySet().size()
3118             + " overlap groups with " + overlapGroups.size()
3119             + " overlapping regions");
3120       }
3121       if (!sidelinedRegions.isEmpty()) {
3122         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
3123         errors.print("---- Table '"  +  this.tableName
3124             + "': sidelined big overlapped regions");
3125         dumpSidelinedRegions(sidelinedRegions);
3126       }
3127       return errors.getErrorList().size() == originalErrorsCount;
3128     }
3129 
3130     private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
3131         throws IOException {
3132       // we parallelize overlap handler for the case we have lots of groups to fix.  We can
3133       // safely assume each group is independent.
3134       List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
3135       List<Future<Void>> rets;
3136       for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3137         //
3138         merges.add(new WorkItemOverlapMerge(overlap, handler));
3139       }
3140       try {
3141         rets = executor.invokeAll(merges);
3142       } catch (InterruptedException e) {
3143         LOG.error("Overlap merges were interrupted", e);
3144         return false;
3145       }
3146       for(int i=0; i<merges.size(); i++) {
3147         WorkItemOverlapMerge work = merges.get(i);
3148         Future<Void> f = rets.get(i);
3149         try {
3150           f.get();
3151         } catch(ExecutionException e) {
3152           LOG.warn("Failed to merge overlap group" + work, e.getCause());
3153         } catch (InterruptedException e) {
3154           LOG.error("Waiting for overlap merges was interrupted", e);
3155           return false;
3156         }
3157       }
3158       return true;
3159     }
3160 
3161     /**
3162      * This dumps data in a visually reasonable way for visual debugging
3163      *
3164      * @param splits
3165      * @param regions
3166      */
3167     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
3168       // we display this way because the last end key should be displayed as well.
3169       StringBuilder sb = new StringBuilder();
3170       for (byte[] k : splits) {
3171         sb.setLength(0); // clear out existing buffer, if any.
3172         sb.append(Bytes.toStringBinary(k) + ":\t");
3173         for (HbckInfo r : regions.get(k)) {
3174           sb.append("[ "+ r.toString() + ", "
3175               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
3176         }
3177         errors.print(sb.toString());
3178       }
3179     }
3180   }
3181 
3182   public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
3183     // we display this way because the last end key should be displayed as
3184     // well.
3185     for (byte[] k : regions.keySet()) {
3186       errors.print(Bytes.toStringBinary(k) + ":");
3187       for (HbckInfo r : regions.get(k)) {
3188         errors.print("[ " + r.toString() + ", "
3189             + Bytes.toStringBinary(r.getEndKey()) + "]");
3190       }
3191       errors.print("----");
3192     }
3193   }
3194 
3195   public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
3196     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
3197       TableName tableName = entry.getValue().getTableName();
3198       Path path = entry.getKey();
3199       errors.print("This sidelined region dir should be bulk loaded: "
3200         + path.toString());
3201       errors.print("Bulk load command looks like: "
3202         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
3203         + path.toUri().getPath() + " "+ tableName);
3204     }
3205   }
3206 
3207   public Multimap<byte[], HbckInfo> getOverlapGroups(
3208       TableName table) {
3209     TableInfo ti = tablesInfo.get(table);
3210     return ti.overlapGroups;
3211   }
3212 
3213   /**
3214    * Return a list of user-space table names whose metadata have not been
3215    * modified in the last few milliseconds specified by timelag
3216    * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
3217    * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
3218    * milliseconds specified by timelag, then the table is a candidate to be returned.
3219    * @return tables that have not been modified recently
3220    * @throws IOException if an error is encountered
3221    */
3222   HTableDescriptor[] getTables(AtomicInteger numSkipped) {
3223     List<TableName> tableNames = new ArrayList<TableName>();
3224     long now = EnvironmentEdgeManager.currentTime();
3225 
3226     for (HbckInfo hbi : regionInfoMap.values()) {
3227       MetaEntry info = hbi.metaEntry;
3228 
3229       // if the start key is zero, then we have found the first region of a table.
3230       // pick only those tables that were not modified in the last few milliseconds.
3231       if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
3232         if (info.modTime + timelag < now) {
3233           tableNames.add(info.getTable());
3234         } else {
3235           numSkipped.incrementAndGet(); // one more in-flux table
3236         }
3237       }
3238     }
3239     return getHTableDescriptors(tableNames);
3240   }
3241 
3242   HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
3243     HTableDescriptor[] htd = new HTableDescriptor[0];
3244       LOG.info("getHTableDescriptors == tableNames => " + tableNames);
3245     try (Connection conn = ConnectionFactory.createConnection(getConf());
3246         Admin admin = conn.getAdmin()) {
3247       htd = admin.getTableDescriptorsByTableName(tableNames);
3248     } catch (IOException e) {
3249       LOG.debug("Exception getting table descriptors", e);
3250     }
3251     return htd;
3252   }
3253 
3254   /**
3255    * Gets the entry in regionInfo corresponding to the the given encoded
3256    * region name. If the region has not been seen yet, a new entry is added
3257    * and returned.
3258    */
3259   private synchronized HbckInfo getOrCreateInfo(String name) {
3260     HbckInfo hbi = regionInfoMap.get(name);
3261     if (hbi == null) {
3262       hbi = new HbckInfo(null);
3263       regionInfoMap.put(name, hbi);
3264     }
3265     return hbi;
3266   }
3267 
3268   private void checkAndFixTableLocks() throws IOException {
3269     TableLockChecker checker = new TableLockChecker(createZooKeeperWatcher(), errors);
3270     checker.checkTableLocks();
3271 
3272     if (this.fixTableLocks) {
3273       checker.fixExpiredTableLocks();
3274     }
3275   }
3276 
3277   /**
3278     * Check values in regionInfo for hbase:meta
3279     * Check if zero or more than one regions with hbase:meta are found.
3280     * If there are inconsistencies (i.e. zero or more than one regions
3281     * pretend to be holding the hbase:meta) try to fix that and report an error.
3282     * @throws IOException from HBaseFsckRepair functions
3283     * @throws KeeperException
3284     * @throws InterruptedException
3285     */
3286   boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
3287     Map<Integer, HbckInfo> metaRegions = new HashMap<Integer, HbckInfo>();
3288     for (HbckInfo value : regionInfoMap.values()) {
3289       if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
3290         metaRegions.put(value.getReplicaId(), value);
3291       }
3292     }
3293     int metaReplication = admin.getTableDescriptor(TableName.META_TABLE_NAME)
3294         .getRegionReplication();
3295     boolean noProblem = true;
3296     // There will be always entries in regionInfoMap corresponding to hbase:meta & its replicas
3297     // Check the deployed servers. It should be exactly one server for each replica.
3298     for (int i = 0; i < metaReplication; i++) {
3299       HbckInfo metaHbckInfo = metaRegions.remove(i);
3300       List<ServerName> servers = new ArrayList<ServerName>();
3301       if (metaHbckInfo != null) {
3302         servers = metaHbckInfo.deployedOn;
3303       }
3304       if (servers.size() != 1) {
3305         noProblem = false;
3306         if (servers.size() == 0) {
3307           assignMetaReplica(i);
3308         } else if (servers.size() > 1) {
3309           errors
3310           .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId " +
3311                        metaHbckInfo.getReplicaId() + " is found on more than one region.");
3312           if (shouldFixAssignments()) {
3313             errors.print("Trying to fix a problem with hbase:meta, replicaId " +
3314                          metaHbckInfo.getReplicaId() +"..");
3315             setShouldRerun();
3316             // try fix it (treat is a dupe assignment)
3317             HBaseFsckRepair.fixMultiAssignment(connection, metaHbckInfo.metaEntry, servers);
3318           }
3319         }
3320       }
3321     }
3322     // unassign whatever is remaining in metaRegions. They are excess replicas.
3323     for (Map.Entry<Integer, HbckInfo> entry : metaRegions.entrySet()) {
3324       noProblem = false;
3325       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
3326           "hbase:meta replicas are deployed in excess. Configured " + metaReplication +
3327           ", deployed " + metaRegions.size());
3328       if (shouldFixAssignments()) {
3329         errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() +
3330             " of hbase:meta..");
3331         setShouldRerun();
3332         unassignMetaReplica(entry.getValue());
3333       }
3334     }
3335     // if noProblem is false, rerun hbck with hopefully fixed META
3336     // if noProblem is true, no errors, so continue normally
3337     return noProblem;
3338   }
3339 
3340   private void unassignMetaReplica(HbckInfo hi) throws IOException, InterruptedException,
3341   KeeperException {
3342     undeployRegions(hi);
3343     ZooKeeperWatcher zkw = createZooKeeperWatcher();
3344     try {
3345       ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(hi.metaEntry.getReplicaId()));
3346     } finally {
3347       zkw.close();
3348     }
3349   }
3350 
3351   private void assignMetaReplica(int replicaId)
3352       throws IOException, KeeperException, InterruptedException {
3353     errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " +
3354         replicaId +" is not found on any region.");
3355     if (shouldFixAssignments()) {
3356       errors.print("Trying to fix a problem with hbase:meta..");
3357       setShouldRerun();
3358       // try to fix it (treat it as unassigned region)
3359       HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
3360           HRegionInfo.FIRST_META_REGIONINFO, replicaId);
3361       HBaseFsckRepair.fixUnassigned(admin, h);
3362       HBaseFsckRepair.waitUntilAssigned(admin, h);
3363     }
3364   }
3365 
3366   /**
3367    * Scan hbase:meta, adding all regions found to the regionInfo map.
3368    * @throws IOException if an error is encountered
3369    */
3370   boolean loadMetaEntries() throws IOException {
3371     MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
3372       int countRecord = 1;
3373 
3374       // comparator to sort KeyValues with latest modtime
3375       final Comparator<Cell> comp = new Comparator<Cell>() {
3376         @Override
3377         public int compare(Cell k1, Cell k2) {
3378           return (int)(k1.getTimestamp() - k2.getTimestamp());
3379         }
3380       };
3381 
3382       @Override
3383       public boolean visit(Result result) throws IOException {
3384         try {
3385 
3386           // record the latest modification of this META record
3387           long ts =  Collections.max(result.listCells(), comp).getTimestamp();
3388           RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
3389           if (rl == null) {
3390             emptyRegionInfoQualifiers.add(result);
3391             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3392               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3393             return true;
3394           }
3395           ServerName sn = null;
3396           if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
3397               rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
3398             emptyRegionInfoQualifiers.add(result);
3399             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3400               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3401             return true;
3402           }
3403           HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
3404           if (!(isTableIncluded(hri.getTable())
3405               || hri.isMetaRegion())) {
3406             return true;
3407           }
3408           PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
3409           for (HRegionLocation h : rl.getRegionLocations()) {
3410             if (h == null || h.getRegionInfo() == null) {
3411               continue;
3412             }
3413             sn = h.getServerName();
3414             hri = h.getRegionInfo();
3415 
3416             MetaEntry m = null;
3417             if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
3418               m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
3419             } else {
3420               m = new MetaEntry(hri, sn, ts, null, null);
3421             }
3422             HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
3423             if (previous == null) {
3424               regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
3425             } else if (previous.metaEntry == null) {
3426               previous.metaEntry = m;
3427             } else {
3428               throw new IOException("Two entries in hbase:meta are same " + previous);
3429             }
3430           }
3431           PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
3432           for (HRegionInfo mergeRegion : new HRegionInfo[] {
3433               mergeRegions.getFirst(), mergeRegions.getSecond() }) {
3434             if (mergeRegion != null) {
3435               // This region is already been merged
3436               HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
3437               hbInfo.setMerged(true);
3438             }
3439           }
3440 
3441           // show proof of progress to the user, once for every 100 records.
3442           if (countRecord % 100 == 0) {
3443             errors.progress();
3444           }
3445           countRecord++;
3446           return true;
3447         } catch (RuntimeException e) {
3448           LOG.error("Result=" + result);
3449           throw e;
3450         }
3451       }
3452     };
3453     if (!checkMetaOnly) {
3454       // Scan hbase:meta to pick up user regions
3455       MetaTableAccessor.fullScanRegions(connection, visitor);
3456     }
3457 
3458     errors.print("");
3459     return true;
3460   }
3461 
3462   /**
3463    * Stores the regioninfo entries scanned from META
3464    */
3465   static class MetaEntry extends HRegionInfo {
3466     ServerName regionServer;   // server hosting this region
3467     long modTime;          // timestamp of most recent modification metadata
3468     HRegionInfo splitA, splitB; //split daughters
3469 
3470     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3471       this(rinfo, regionServer, modTime, null, null);
3472     }
3473 
3474     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3475         HRegionInfo splitA, HRegionInfo splitB) {
3476       super(rinfo);
3477       this.regionServer = regionServer;
3478       this.modTime = modTime;
3479       this.splitA = splitA;
3480       this.splitB = splitB;
3481     }
3482 
3483     @Override
3484     public boolean equals(Object o) {
3485       boolean superEq = super.equals(o);
3486       if (!superEq) {
3487         return superEq;
3488       }
3489 
3490       MetaEntry me = (MetaEntry) o;
3491       if (!regionServer.equals(me.regionServer)) {
3492         return false;
3493       }
3494       return (modTime == me.modTime);
3495     }
3496 
3497     @Override
3498     public int hashCode() {
3499       int hash = Arrays.hashCode(getRegionName());
3500       hash ^= getRegionId();
3501       hash ^= Arrays.hashCode(getStartKey());
3502       hash ^= Arrays.hashCode(getEndKey());
3503       hash ^= Boolean.valueOf(isOffline()).hashCode();
3504       hash ^= getTable().hashCode();
3505       if (regionServer != null) {
3506         hash ^= regionServer.hashCode();
3507       }
3508       hash ^= modTime;
3509       return hash;
3510     }
3511   }
3512 
3513   /**
3514    * Stores the regioninfo entries from HDFS
3515    */
3516   static class HdfsEntry {
3517     HRegionInfo hri;
3518     Path hdfsRegionDir = null;
3519     long hdfsRegionDirModTime  = 0;
3520     boolean hdfsRegioninfoFilePresent = false;
3521     boolean hdfsOnlyEdits = false;
3522   }
3523 
3524   /**
3525    * Stores the regioninfo retrieved from Online region servers.
3526    */
3527   static class OnlineEntry {
3528     HRegionInfo hri;
3529     ServerName hsa;
3530 
3531     @Override
3532     public String toString() {
3533       return hsa.toString() + ";" + hri.getRegionNameAsString();
3534     }
3535   }
3536 
3537   /**
3538    * Maintain information about a particular region.  It gathers information
3539    * from three places -- HDFS, META, and region servers.
3540    */
3541   public static class HbckInfo implements KeyRange {
3542     private MetaEntry metaEntry = null; // info in META
3543     private HdfsEntry hdfsEntry = null; // info in HDFS
3544     private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
3545     private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
3546     private boolean skipChecks = false; // whether to skip further checks to this region info.
3547     private boolean isMerged = false;// whether this region has already been merged into another one
3548     private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3549     private HRegionInfo primaryHRIForDeployedReplica = null;
3550 
3551     HbckInfo(MetaEntry metaEntry) {
3552       this.metaEntry = metaEntry;
3553     }
3554 
3555     public int getReplicaId() {
3556       if (metaEntry != null) return metaEntry.getReplicaId();
3557       return deployedReplicaId;
3558     }
3559 
3560     public synchronized void addServer(HRegionInfo hri, ServerName server) {
3561       OnlineEntry rse = new OnlineEntry() ;
3562       rse.hri = hri;
3563       rse.hsa = server;
3564       this.deployedEntries.add(rse);
3565       this.deployedOn.add(server);
3566       // save the replicaId that we see deployed in the cluster
3567       this.deployedReplicaId = hri.getReplicaId();
3568       this.primaryHRIForDeployedReplica =
3569           RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3570     }
3571 
3572     @Override
3573     public synchronized String toString() {
3574       StringBuilder sb = new StringBuilder();
3575       sb.append("{ meta => ");
3576       sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3577       sb.append( ", hdfs => " + getHdfsRegionDir());
3578       sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3579       sb.append( ", replicaId => " + getReplicaId());
3580       sb.append(" }");
3581       return sb.toString();
3582     }
3583 
3584     @Override
3585     public byte[] getStartKey() {
3586       if (this.metaEntry != null) {
3587         return this.metaEntry.getStartKey();
3588       } else if (this.hdfsEntry != null) {
3589         return this.hdfsEntry.hri.getStartKey();
3590       } else {
3591         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3592         return null;
3593       }
3594     }
3595 
3596     @Override
3597     public byte[] getEndKey() {
3598       if (this.metaEntry != null) {
3599         return this.metaEntry.getEndKey();
3600       } else if (this.hdfsEntry != null) {
3601         return this.hdfsEntry.hri.getEndKey();
3602       } else {
3603         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3604         return null;
3605       }
3606     }
3607 
3608     public TableName getTableName() {
3609       if (this.metaEntry != null) {
3610         return this.metaEntry.getTable();
3611       } else if (this.hdfsEntry != null) {
3612         // we are only guaranteed to have a path and not an HRI for hdfsEntry,
3613         // so we get the name from the Path
3614         Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3615         return FSUtils.getTableName(tableDir);
3616       } else {
3617         // return the info from the first online/deployed hri
3618         for (OnlineEntry e : deployedEntries) {
3619           return e.hri.getTable();
3620         }
3621         return null;
3622       }
3623     }
3624 
3625     public String getRegionNameAsString() {
3626       if (metaEntry != null) {
3627         return metaEntry.getRegionNameAsString();
3628       } else if (hdfsEntry != null) {
3629         if (hdfsEntry.hri != null) {
3630           return hdfsEntry.hri.getRegionNameAsString();
3631         }
3632       } else {
3633         // return the info from the first online/deployed hri
3634         for (OnlineEntry e : deployedEntries) {
3635           return e.hri.getRegionNameAsString();
3636         }
3637       }
3638       return null;
3639     }
3640 
3641     public byte[] getRegionName() {
3642       if (metaEntry != null) {
3643         return metaEntry.getRegionName();
3644       } else if (hdfsEntry != null) {
3645         return hdfsEntry.hri.getRegionName();
3646       } else {
3647         // return the info from the first online/deployed hri
3648         for (OnlineEntry e : deployedEntries) {
3649           return e.hri.getRegionName();
3650         }
3651         return null;
3652       }
3653     }
3654 
3655     public HRegionInfo getPrimaryHRIForDeployedReplica() {
3656       return primaryHRIForDeployedReplica;
3657     }
3658 
3659     Path getHdfsRegionDir() {
3660       if (hdfsEntry == null) {
3661         return null;
3662       }
3663       return hdfsEntry.hdfsRegionDir;
3664     }
3665 
3666     boolean containsOnlyHdfsEdits() {
3667       if (hdfsEntry == null) {
3668         return false;
3669       }
3670       return hdfsEntry.hdfsOnlyEdits;
3671     }
3672 
3673     boolean isHdfsRegioninfoPresent() {
3674       if (hdfsEntry == null) {
3675         return false;
3676       }
3677       return hdfsEntry.hdfsRegioninfoFilePresent;
3678     }
3679 
3680     long getModTime() {
3681       if (hdfsEntry == null) {
3682         return 0;
3683       }
3684       return hdfsEntry.hdfsRegionDirModTime;
3685     }
3686 
3687     HRegionInfo getHdfsHRI() {
3688       if (hdfsEntry == null) {
3689         return null;
3690       }
3691       return hdfsEntry.hri;
3692     }
3693 
3694     public void setSkipChecks(boolean skipChecks) {
3695       this.skipChecks = skipChecks;
3696     }
3697 
3698     public boolean isSkipChecks() {
3699       return skipChecks;
3700     }
3701 
3702     public void setMerged(boolean isMerged) {
3703       this.isMerged = isMerged;
3704     }
3705 
3706     public boolean isMerged() {
3707       return this.isMerged;
3708     }
3709   }
3710 
3711   final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3712     @Override
3713     public int compare(HbckInfo l, HbckInfo r) {
3714       if (l == r) {
3715         // same instance
3716         return 0;
3717       }
3718 
3719       int tableCompare = l.getTableName().compareTo(r.getTableName());
3720       if (tableCompare != 0) {
3721         return tableCompare;
3722       }
3723 
3724       int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3725           l.getStartKey(), r.getStartKey());
3726       if (startComparison != 0) {
3727         return startComparison;
3728       }
3729 
3730       // Special case for absolute endkey
3731       byte[] endKey = r.getEndKey();
3732       endKey = (endKey.length == 0) ? null : endKey;
3733       byte[] endKey2 = l.getEndKey();
3734       endKey2 = (endKey2.length == 0) ? null : endKey2;
3735       int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3736           endKey2,  endKey);
3737 
3738       if (endComparison != 0) {
3739         return endComparison;
3740       }
3741 
3742       // use regionId as tiebreaker.
3743       // Null is considered after all possible values so make it bigger.
3744       if (l.hdfsEntry == null && r.hdfsEntry == null) {
3745         return 0;
3746       }
3747       if (l.hdfsEntry == null && r.hdfsEntry != null) {
3748         return 1;
3749       }
3750       // l.hdfsEntry must not be null
3751       if (r.hdfsEntry == null) {
3752         return -1;
3753       }
3754       // both l.hdfsEntry and r.hdfsEntry must not be null.
3755       return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3756     }
3757   };
3758 
3759   /**
3760    * Prints summary of all tables found on the system.
3761    */
3762   private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3763     StringBuilder sb = new StringBuilder();
3764     int numOfSkippedRegions;
3765     errors.print("Summary:");
3766     for (TableInfo tInfo : tablesInfo.values()) {
3767       numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName())) ?
3768           skippedRegions.get(tInfo.getName()).size() : 0;
3769 
3770       if (errors.tableHasErrors(tInfo)) {
3771         errors.print("Table " + tInfo.getName() + " is inconsistent.");
3772       } else if (numOfSkippedRegions > 0){
3773         errors.print("Table " + tInfo.getName() + " is okay (with "
3774           + numOfSkippedRegions + " skipped regions).");
3775       }
3776       else {
3777         errors.print("Table " + tInfo.getName() + " is okay.");
3778       }
3779       errors.print("    Number of regions: " + tInfo.getNumRegions());
3780       if (numOfSkippedRegions > 0) {
3781         Set<String> skippedRegionStrings = skippedRegions.get(tInfo.getName());
3782         System.out.println("    Number of skipped regions: " + numOfSkippedRegions);
3783         System.out.println("      List of skipped regions:");
3784         for(String sr : skippedRegionStrings) {
3785           System.out.println("        " + sr);
3786         }
3787       }
3788       sb.setLength(0); // clear out existing buffer, if any.
3789       sb.append("    Deployed on: ");
3790       for (ServerName server : tInfo.deployedOn) {
3791         sb.append(" " + server.toString());
3792       }
3793       errors.print(sb.toString());
3794     }
3795   }
3796 
3797   static ErrorReporter getErrorReporter(
3798       final Configuration conf) throws ClassNotFoundException {
3799     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3800     return ReflectionUtils.newInstance(reporter, conf);
3801   }
3802 
3803   public interface ErrorReporter {
3804     enum ERROR_CODE {
3805       UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3806       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META,
3807       NOT_DEPLOYED,
3808       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3809       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3810       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3811       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3812       WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR, ORPHAN_TABLE_STATE,
3813       NO_TABLE_STATE
3814     }
3815     void clear();
3816     void report(String message);
3817     void reportError(String message);
3818     void reportError(ERROR_CODE errorCode, String message);
3819     void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3820     void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3821     void reportError(
3822       ERROR_CODE errorCode,
3823       String message,
3824       TableInfo table,
3825       HbckInfo info1,
3826       HbckInfo info2
3827     );
3828     int summarize();
3829     void detail(String details);
3830     ArrayList<ERROR_CODE> getErrorList();
3831     void progress();
3832     void print(String message);
3833     void resetErrors();
3834     boolean tableHasErrors(TableInfo table);
3835   }
3836 
3837   static class PrintingErrorReporter implements ErrorReporter {
3838     public int errorCount = 0;
3839     private int showProgress;
3840     // How frequently calls to progress() will create output
3841     private static final int progressThreshold = 100;
3842 
3843     Set<TableInfo> errorTables = new HashSet<TableInfo>();
3844 
3845     // for use by unit tests to verify which errors were discovered
3846     private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3847 
3848     @Override
3849     public void clear() {
3850       errorTables.clear();
3851       errorList.clear();
3852       errorCount = 0;
3853     }
3854 
3855     @Override
3856     public synchronized void reportError(ERROR_CODE errorCode, String message) {
3857       if (errorCode == ERROR_CODE.WRONG_USAGE) {
3858         System.err.println(message);
3859         return;
3860       }
3861 
3862       errorList.add(errorCode);
3863       if (!summary) {
3864         System.out.println("ERROR: " + message);
3865       }
3866       errorCount++;
3867       showProgress = 0;
3868     }
3869 
3870     @Override
3871     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
3872       errorTables.add(table);
3873       reportError(errorCode, message);
3874     }
3875 
3876     @Override
3877     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3878                                          HbckInfo info) {
3879       errorTables.add(table);
3880       String reference = "(region " + info.getRegionNameAsString() + ")";
3881       reportError(errorCode, reference + " " + message);
3882     }
3883 
3884     @Override
3885     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3886                                          HbckInfo info1, HbckInfo info2) {
3887       errorTables.add(table);
3888       String reference = "(regions " + info1.getRegionNameAsString()
3889           + " and " + info2.getRegionNameAsString() + ")";
3890       reportError(errorCode, reference + " " + message);
3891     }
3892 
3893     @Override
3894     public synchronized void reportError(String message) {
3895       reportError(ERROR_CODE.UNKNOWN, message);
3896     }
3897 
3898     /**
3899      * Report error information, but do not increment the error count.  Intended for cases
3900      * where the actual error would have been reported previously.
3901      * @param message
3902      */
3903     @Override
3904     public synchronized void report(String message) {
3905       if (! summary) {
3906         System.out.println("ERROR: " + message);
3907       }
3908       showProgress = 0;
3909     }
3910 
3911     @Override
3912     public synchronized int summarize() {
3913       System.out.println(Integer.toString(errorCount) +
3914                          " inconsistencies detected.");
3915       if (errorCount == 0) {
3916         System.out.println("Status: OK");
3917         return 0;
3918       } else {
3919         System.out.println("Status: INCONSISTENT");
3920         return -1;
3921       }
3922     }
3923 
3924     @Override
3925     public ArrayList<ERROR_CODE> getErrorList() {
3926       return errorList;
3927     }
3928 
3929     @Override
3930     public synchronized void print(String message) {
3931       if (!summary) {
3932         System.out.println(message);
3933       }
3934     }
3935 
3936     @Override
3937     public boolean tableHasErrors(TableInfo table) {
3938       return errorTables.contains(table);
3939     }
3940 
3941     @Override
3942     public void resetErrors() {
3943       errorCount = 0;
3944     }
3945 
3946     @Override
3947     public synchronized void detail(String message) {
3948       if (details) {
3949         System.out.println(message);
3950       }
3951       showProgress = 0;
3952     }
3953 
3954     @Override
3955     public synchronized void progress() {
3956       if (showProgress++ == progressThreshold) {
3957         if (!summary) {
3958           System.out.print(".");
3959         }
3960         showProgress = 0;
3961       }
3962     }
3963   }
3964 
3965   /**
3966    * Contact a region server and get all information from it
3967    */
3968   static class WorkItemRegion implements Callable<Void> {
3969     private HBaseFsck hbck;
3970     private ServerName rsinfo;
3971     private ErrorReporter errors;
3972     private HConnection connection;
3973 
3974     WorkItemRegion(HBaseFsck hbck, ServerName info,
3975                    ErrorReporter errors, HConnection connection) {
3976       this.hbck = hbck;
3977       this.rsinfo = info;
3978       this.errors = errors;
3979       this.connection = connection;
3980     }
3981 
3982     @Override
3983     public synchronized Void call() throws IOException {
3984       errors.progress();
3985       try {
3986         BlockingInterface server = connection.getAdmin(rsinfo);
3987 
3988         // list all online regions from this region server
3989         List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
3990         regions = filterRegions(regions);
3991 
3992         if (details) {
3993           errors.detail("RegionServer: " + rsinfo.getServerName() +
3994                            " number of regions: " + regions.size());
3995           for (HRegionInfo rinfo: regions) {
3996             errors.detail("  " + rinfo.getRegionNameAsString() +
3997                              " id: " + rinfo.getRegionId() +
3998                              " encoded_name: " + rinfo.getEncodedName() +
3999                              " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
4000                              " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
4001           }
4002         }
4003 
4004         // check to see if the existence of this region matches the region in META
4005         for (HRegionInfo r:regions) {
4006           HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
4007           hbi.addServer(r, rsinfo);
4008         }
4009       } catch (IOException e) {          // unable to connect to the region server.
4010         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
4011           " Unable to fetch region information. " + e);
4012         throw e;
4013       }
4014       return null;
4015     }
4016 
4017     private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
4018       List<HRegionInfo> ret = Lists.newArrayList();
4019       for (HRegionInfo hri : regions) {
4020         if (hri.isMetaTable() || (!hbck.checkMetaOnly
4021             && hbck.isTableIncluded(hri.getTable()))) {
4022           ret.add(hri);
4023         }
4024       }
4025       return ret;
4026     }
4027   }
4028 
4029   /**
4030    * Contact hdfs and get all information about specified table directory into
4031    * regioninfo list.
4032    */
4033   static class WorkItemHdfsDir implements Callable<Void> {
4034     private HBaseFsck hbck;
4035     private FileStatus tableDir;
4036     private ErrorReporter errors;
4037     private FileSystem fs;
4038 
4039     WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
4040                     FileStatus status) {
4041       this.hbck = hbck;
4042       this.fs = fs;
4043       this.tableDir = status;
4044       this.errors = errors;
4045     }
4046 
4047     @Override
4048     public synchronized Void call() throws IOException {
4049       try {
4050         // level 2: <HBASE_DIR>/<table>/*
4051         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
4052         for (FileStatus regionDir : regionDirs) {
4053           errors.progress();
4054           String encodedName = regionDir.getPath().getName();
4055           // ignore directories that aren't hexadecimal
4056           if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
4057             continue;
4058           }
4059 
4060           LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
4061           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
4062           HdfsEntry he = new HdfsEntry();
4063           synchronized (hbi) {
4064             if (hbi.getHdfsRegionDir() != null) {
4065               errors.print("Directory " + encodedName + " duplicate??" +
4066                            hbi.getHdfsRegionDir());
4067             }
4068 
4069             he.hdfsRegionDir = regionDir.getPath();
4070             he.hdfsRegionDirModTime = regionDir.getModificationTime();
4071             Path regioninfoFile = new Path(he.hdfsRegionDir, HRegionFileSystem.REGION_INFO_FILE);
4072             he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
4073             // we add to orphan list when we attempt to read .regioninfo
4074 
4075             // Set a flag if this region contains only edits
4076             // This is special case if a region is left after split
4077             he.hdfsOnlyEdits = true;
4078             FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
4079             Path ePath = WALSplitter.getRegionDirRecoveredEditsDir(regionDir.getPath());
4080             for (FileStatus subDir : subDirs) {
4081               errors.progress();
4082               String sdName = subDir.getPath().getName();
4083               if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
4084                 he.hdfsOnlyEdits = false;
4085                 break;
4086               }
4087             }
4088             hbi.hdfsEntry = he;
4089           }
4090         }
4091       } catch (IOException e) {
4092         // unable to connect to the region server.
4093         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
4094             + tableDir.getPath().getName()
4095             + " Unable to fetch region information. " + e);
4096         throw e;
4097       }
4098       return null;
4099     }
4100   }
4101 
4102   /**
4103    * Contact hdfs and get all information about specified table directory into
4104    * regioninfo list.
4105    */
4106   static class WorkItemHdfsRegionInfo implements Callable<Void> {
4107     private HbckInfo hbi;
4108     private HBaseFsck hbck;
4109     private ErrorReporter errors;
4110 
4111     WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
4112       this.hbi = hbi;
4113       this.hbck = hbck;
4114       this.errors = errors;
4115     }
4116 
4117     @Override
4118     public synchronized Void call() throws IOException {
4119       // only load entries that haven't been loaded yet.
4120       if (hbi.getHdfsHRI() == null) {
4121         try {
4122           errors.progress();
4123           hbck.loadHdfsRegioninfo(hbi);
4124         } catch (IOException ioe) {
4125           String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
4126               + hbi.getTableName() + " in hdfs dir "
4127               + hbi.getHdfsRegionDir()
4128               + "!  It may be an invalid format or version file.  Treating as "
4129               + "an orphaned regiondir.";
4130           errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
4131           try {
4132             hbck.debugLsr(hbi.getHdfsRegionDir());
4133           } catch (IOException ioe2) {
4134             LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
4135             throw ioe2;
4136           }
4137           hbck.orphanHdfsDirs.add(hbi);
4138           throw ioe;
4139         }
4140       }
4141       return null;
4142     }
4143   };
4144 
4145   /**
4146    * Display the full report from fsck. This displays all live and dead region
4147    * servers, and all known regions.
4148    */
4149   public static void setDisplayFullReport() {
4150     details = true;
4151   }
4152 
4153   public static void setNoLock() {
4154     useLock = false;
4155   }
4156 
4157   public static void setNoBalacerSwitch() {
4158     switchBalancer = false;
4159   }
4160 
4161   /**
4162    * Set summary mode.
4163    * Print only summary of the tables and status (OK or INCONSISTENT)
4164    */
4165   static void setSummary() {
4166     summary = true;
4167   }
4168 
4169   /**
4170    * Set hbase:meta check mode.
4171    * Print only info about hbase:meta table deployment/state
4172    */
4173   void setCheckMetaOnly() {
4174     checkMetaOnly = true;
4175   }
4176 
4177   /**
4178    * Set region boundaries check mode.
4179    */
4180   void setRegionBoundariesCheck() {
4181     checkRegionBoundaries = true;
4182   }
4183 
4184   /**
4185    * Set table locks fix mode.
4186    * Delete table locks held for a long time
4187    */
4188   public void setFixTableLocks(boolean shouldFix) {
4189     fixTableLocks = shouldFix;
4190     fixAny |= shouldFix;
4191   }
4192 
4193   /**
4194    * Check if we should rerun fsck again. This checks if we've tried to
4195    * fix something and we should rerun fsck tool again.
4196    * Display the full report from fsck. This displays all live and dead
4197    * region servers, and all known regions.
4198    */
4199   void setShouldRerun() {
4200     rerun = true;
4201   }
4202 
4203   boolean shouldRerun() {
4204     return rerun;
4205   }
4206 
4207   /**
4208    * Fix inconsistencies found by fsck. This should try to fix errors (if any)
4209    * found by fsck utility.
4210    */
4211   public void setFixAssignments(boolean shouldFix) {
4212     fixAssignments = shouldFix;
4213     fixAny |= shouldFix;
4214   }
4215 
4216   boolean shouldFixAssignments() {
4217     return fixAssignments;
4218   }
4219 
4220   public void setFixMeta(boolean shouldFix) {
4221     fixMeta = shouldFix;
4222     fixAny |= shouldFix;
4223   }
4224 
4225   boolean shouldFixMeta() {
4226     return fixMeta;
4227   }
4228 
4229   public void setFixEmptyMetaCells(boolean shouldFix) {
4230     fixEmptyMetaCells = shouldFix;
4231     fixAny |= shouldFix;
4232   }
4233 
4234   boolean shouldFixEmptyMetaCells() {
4235     return fixEmptyMetaCells;
4236   }
4237 
4238   public void setCheckHdfs(boolean checking) {
4239     checkHdfs = checking;
4240   }
4241 
4242   boolean shouldCheckHdfs() {
4243     return checkHdfs;
4244   }
4245 
4246   public void setFixHdfsHoles(boolean shouldFix) {
4247     fixHdfsHoles = shouldFix;
4248     fixAny |= shouldFix;
4249   }
4250 
4251   boolean shouldFixHdfsHoles() {
4252     return fixHdfsHoles;
4253   }
4254 
4255   public void setFixTableOrphans(boolean shouldFix) {
4256     fixTableOrphans = shouldFix;
4257     fixAny |= shouldFix;
4258   }
4259 
4260   boolean shouldFixTableOrphans() {
4261     return fixTableOrphans;
4262   }
4263 
4264   public void setFixHdfsOverlaps(boolean shouldFix) {
4265     fixHdfsOverlaps = shouldFix;
4266     fixAny |= shouldFix;
4267   }
4268 
4269   boolean shouldFixHdfsOverlaps() {
4270     return fixHdfsOverlaps;
4271   }
4272 
4273   public void setFixHdfsOrphans(boolean shouldFix) {
4274     fixHdfsOrphans = shouldFix;
4275     fixAny |= shouldFix;
4276   }
4277 
4278   boolean shouldFixHdfsOrphans() {
4279     return fixHdfsOrphans;
4280   }
4281 
4282   public void setFixVersionFile(boolean shouldFix) {
4283     fixVersionFile = shouldFix;
4284     fixAny |= shouldFix;
4285   }
4286 
4287   public boolean shouldFixVersionFile() {
4288     return fixVersionFile;
4289   }
4290 
4291   public void setSidelineBigOverlaps(boolean sbo) {
4292     this.sidelineBigOverlaps = sbo;
4293   }
4294 
4295   public boolean shouldSidelineBigOverlaps() {
4296     return sidelineBigOverlaps;
4297   }
4298 
4299   public void setFixSplitParents(boolean shouldFix) {
4300     fixSplitParents = shouldFix;
4301     fixAny |= shouldFix;
4302   }
4303 
4304   boolean shouldFixSplitParents() {
4305     return fixSplitParents;
4306   }
4307 
4308   public void setFixReferenceFiles(boolean shouldFix) {
4309     fixReferenceFiles = shouldFix;
4310     fixAny |= shouldFix;
4311   }
4312 
4313   boolean shouldFixReferenceFiles() {
4314     return fixReferenceFiles;
4315   }
4316 
4317   public boolean shouldIgnorePreCheckPermission() {
4318     return !fixAny || ignorePreCheckPermission;
4319   }
4320 
4321   public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
4322     this.ignorePreCheckPermission = ignorePreCheckPermission;
4323   }
4324 
4325   /**
4326    * @param mm maximum number of regions to merge into a single region.
4327    */
4328   public void setMaxMerge(int mm) {
4329     this.maxMerge = mm;
4330   }
4331 
4332   public int getMaxMerge() {
4333     return maxMerge;
4334   }
4335 
4336   public void setMaxOverlapsToSideline(int mo) {
4337     this.maxOverlapsToSideline = mo;
4338   }
4339 
4340   public int getMaxOverlapsToSideline() {
4341     return maxOverlapsToSideline;
4342   }
4343 
4344   /**
4345    * Only check/fix tables specified by the list,
4346    * Empty list means all tables are included.
4347    */
4348   boolean isTableIncluded(TableName table) {
4349     return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
4350   }
4351 
4352   public void includeTable(TableName table) {
4353     tablesIncluded.add(table);
4354   }
4355 
4356   Set<TableName> getIncludedTables() {
4357     return new HashSet<TableName>(tablesIncluded);
4358   }
4359 
4360   /**
4361    * We are interested in only those tables that have not changed their state in
4362    * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
4363    * @param seconds - the time in seconds
4364    */
4365   public void setTimeLag(long seconds) {
4366     timelag = seconds * 1000; // convert to milliseconds
4367   }
4368 
4369   /**
4370    *
4371    * @param sidelineDir - HDFS path to sideline data
4372    */
4373   public void setSidelineDir(String sidelineDir) {
4374     this.sidelineDir = new Path(sidelineDir);
4375   }
4376 
4377   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
4378     return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
4379   }
4380 
4381   public HFileCorruptionChecker getHFilecorruptionChecker() {
4382     return hfcc;
4383   }
4384 
4385   public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
4386     this.hfcc = hfcc;
4387   }
4388 
4389   public void setRetCode(int code) {
4390     this.retcode = code;
4391   }
4392 
4393   public int getRetCode() {
4394     return retcode;
4395   }
4396 
4397   protected HBaseFsck printUsageAndExit() {
4398     StringWriter sw = new StringWriter(2048);
4399     PrintWriter out = new PrintWriter(sw);
4400     out.println("Usage: fsck [opts] {only tables}");
4401     out.println(" where [opts] are:");
4402     out.println("   -help Display help options (this)");
4403     out.println("   -details Display full report of all regions.");
4404     out.println("   -timelag <timeInSeconds>  Process only regions that " +
4405                        " have not experienced any metadata updates in the last " +
4406                        " <timeInSeconds> seconds.");
4407     out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
4408         " before checking if the fix worked if run with -fix");
4409     out.println("   -summary Print only summary of the tables and status.");
4410     out.println("   -metaonly Only check the state of the hbase:meta table.");
4411     out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
4412     out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
4413     out.println("   -noLock Turn off using the hdfs lock file.");
4414     out.println("   -noBalancerSwitch Don't switch the balancer off.");
4415 
4416     out.println("");
4417     out.println("  Metadata Repair options: (expert features, use with caution!)");
4418     out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
4419     out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
4420     out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
4421     out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
4422         + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
4423     out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
4424     out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
4425     out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
4426     out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
4427     out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
4428     out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
4429     out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
4430     out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
4431     out.println("   -fixSplitParents  Try to force offline split parents to be online.");
4432     out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
4433     out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
4434     out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
4435         + " (empty REGIONINFO_QUALIFIER rows)");
4436 
4437     out.println("");
4438     out.println("  Datafile Repair options: (expert features, use with caution!)");
4439     out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
4440     out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
4441 
4442     out.println("");
4443     out.println("  Metadata Repair shortcuts");
4444     out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
4445         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles -fixTableLocks");
4446     out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
4447 
4448     out.println("");
4449     out.println("  Table lock options");
4450     out.println("   -fixTableLocks    Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
4451 
4452     out.flush();
4453     errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
4454 
4455     setRetCode(-2);
4456     return this;
4457   }
4458 
4459   /**
4460    * Main program
4461    *
4462    * @param args
4463    * @throws Exception
4464    */
4465   public static void main(String[] args) throws Exception {
4466     // create a fsck object
4467     Configuration conf = HBaseConfiguration.create();
4468     Path hbasedir = FSUtils.getRootDir(conf);
4469     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
4470     FSUtils.setFsDefault(conf, new Path(defaultFs));
4471     int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
4472     System.exit(ret);
4473   }
4474 
4475   /**
4476    * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
4477    */
4478   static class HBaseFsckTool extends Configured implements Tool {
4479     HBaseFsckTool(Configuration conf) { super(conf); }
4480     @Override
4481     public int run(String[] args) throws Exception {
4482       HBaseFsck hbck = new HBaseFsck(getConf());
4483       hbck.exec(hbck.executor, args);
4484       hbck.close();
4485       return hbck.getRetCode();
4486     }
4487   };
4488 
4489 
4490   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
4491     ServiceException, InterruptedException {
4492     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
4493 
4494     boolean checkCorruptHFiles = false;
4495     boolean sidelineCorruptHFiles = false;
4496 
4497     // Process command-line args.
4498     for (int i = 0; i < args.length; i++) {
4499       String cmd = args[i];
4500       if (cmd.equals("-help") || cmd.equals("-h")) {
4501         return printUsageAndExit();
4502       } else if (cmd.equals("-details")) {
4503         setDisplayFullReport();
4504       } else if (cmd.equals("-noLock")) {
4505         setNoLock();
4506       } else if (cmd.equals("-noBalancerSwitch")) {
4507         setNoBalacerSwitch();
4508       } else if (cmd.equals("-timelag")) {
4509         if (i == args.length - 1) {
4510           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4511           return printUsageAndExit();
4512         }
4513         try {
4514           long timelag = Long.parseLong(args[i+1]);
4515           setTimeLag(timelag);
4516         } catch (NumberFormatException e) {
4517           errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4518           return printUsageAndExit();
4519         }
4520         i++;
4521       } else if (cmd.equals("-sleepBeforeRerun")) {
4522         if (i == args.length - 1) {
4523           errors.reportError(ERROR_CODE.WRONG_USAGE,
4524             "HBaseFsck: -sleepBeforeRerun needs a value.");
4525           return printUsageAndExit();
4526         }
4527         try {
4528           sleepBeforeRerun = Long.parseLong(args[i+1]);
4529         } catch (NumberFormatException e) {
4530           errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4531           return printUsageAndExit();
4532         }
4533         i++;
4534       } else if (cmd.equals("-sidelineDir")) {
4535         if (i == args.length - 1) {
4536           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4537           return printUsageAndExit();
4538         }
4539         i++;
4540         setSidelineDir(args[i]);
4541       } else if (cmd.equals("-fix")) {
4542         errors.reportError(ERROR_CODE.WRONG_USAGE,
4543           "This option is deprecated, please use  -fixAssignments instead.");
4544         setFixAssignments(true);
4545       } else if (cmd.equals("-fixAssignments")) {
4546         setFixAssignments(true);
4547       } else if (cmd.equals("-fixMeta")) {
4548         setFixMeta(true);
4549       } else if (cmd.equals("-noHdfsChecking")) {
4550         setCheckHdfs(false);
4551       } else if (cmd.equals("-fixHdfsHoles")) {
4552         setFixHdfsHoles(true);
4553       } else if (cmd.equals("-fixHdfsOrphans")) {
4554         setFixHdfsOrphans(true);
4555       } else if (cmd.equals("-fixTableOrphans")) {
4556         setFixTableOrphans(true);
4557       } else if (cmd.equals("-fixHdfsOverlaps")) {
4558         setFixHdfsOverlaps(true);
4559       } else if (cmd.equals("-fixVersionFile")) {
4560         setFixVersionFile(true);
4561       } else if (cmd.equals("-sidelineBigOverlaps")) {
4562         setSidelineBigOverlaps(true);
4563       } else if (cmd.equals("-fixSplitParents")) {
4564         setFixSplitParents(true);
4565       } else if (cmd.equals("-ignorePreCheckPermission")) {
4566         setIgnorePreCheckPermission(true);
4567       } else if (cmd.equals("-checkCorruptHFiles")) {
4568         checkCorruptHFiles = true;
4569       } else if (cmd.equals("-sidelineCorruptHFiles")) {
4570         sidelineCorruptHFiles = true;
4571       } else if (cmd.equals("-fixReferenceFiles")) {
4572         setFixReferenceFiles(true);
4573       } else if (cmd.equals("-fixEmptyMetaCells")) {
4574         setFixEmptyMetaCells(true);
4575       } else if (cmd.equals("-repair")) {
4576         // this attempts to merge overlapping hdfs regions, needs testing
4577         // under load
4578         setFixHdfsHoles(true);
4579         setFixHdfsOrphans(true);
4580         setFixMeta(true);
4581         setFixAssignments(true);
4582         setFixHdfsOverlaps(true);
4583         setFixVersionFile(true);
4584         setSidelineBigOverlaps(true);
4585         setFixSplitParents(false);
4586         setCheckHdfs(true);
4587         setFixReferenceFiles(true);
4588         setFixTableLocks(true);
4589       } else if (cmd.equals("-repairHoles")) {
4590         // this will make all missing hdfs regions available but may lose data
4591         setFixHdfsHoles(true);
4592         setFixHdfsOrphans(false);
4593         setFixMeta(true);
4594         setFixAssignments(true);
4595         setFixHdfsOverlaps(false);
4596         setSidelineBigOverlaps(false);
4597         setFixSplitParents(false);
4598         setCheckHdfs(true);
4599       } else if (cmd.equals("-maxOverlapsToSideline")) {
4600         if (i == args.length - 1) {
4601           errors.reportError(ERROR_CODE.WRONG_USAGE,
4602             "-maxOverlapsToSideline needs a numeric value argument.");
4603           return printUsageAndExit();
4604         }
4605         try {
4606           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4607           setMaxOverlapsToSideline(maxOverlapsToSideline);
4608         } catch (NumberFormatException e) {
4609           errors.reportError(ERROR_CODE.WRONG_USAGE,
4610             "-maxOverlapsToSideline needs a numeric value argument.");
4611           return printUsageAndExit();
4612         }
4613         i++;
4614       } else if (cmd.equals("-maxMerge")) {
4615         if (i == args.length - 1) {
4616           errors.reportError(ERROR_CODE.WRONG_USAGE,
4617             "-maxMerge needs a numeric value argument.");
4618           return printUsageAndExit();
4619         }
4620         try {
4621           int maxMerge = Integer.parseInt(args[i+1]);
4622           setMaxMerge(maxMerge);
4623         } catch (NumberFormatException e) {
4624           errors.reportError(ERROR_CODE.WRONG_USAGE,
4625             "-maxMerge needs a numeric value argument.");
4626           return printUsageAndExit();
4627         }
4628         i++;
4629       } else if (cmd.equals("-summary")) {
4630         setSummary();
4631       } else if (cmd.equals("-metaonly")) {
4632         setCheckMetaOnly();
4633       } else if (cmd.equals("-boundaries")) {
4634         setRegionBoundariesCheck();
4635       } else if (cmd.equals("-fixTableLocks")) {
4636         setFixTableLocks(true);
4637       } else if (cmd.startsWith("-")) {
4638         errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4639         return printUsageAndExit();
4640       } else {
4641         includeTable(TableName.valueOf(cmd));
4642         errors.print("Allow checking/fixes for table: " + cmd);
4643       }
4644     }
4645 
4646     errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
4647 
4648     // pre-check current user has FS write permission or not
4649     try {
4650       preCheckPermission();
4651     } catch (AccessDeniedException ace) {
4652       Runtime.getRuntime().exit(-1);
4653     } catch (IOException ioe) {
4654       Runtime.getRuntime().exit(-1);
4655     }
4656 
4657     // do the real work of hbck
4658     connect();
4659 
4660     try {
4661       // if corrupt file mode is on, first fix them since they may be opened later
4662       if (checkCorruptHFiles || sidelineCorruptHFiles) {
4663         LOG.info("Checking all hfiles for corruption");
4664         HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4665         setHFileCorruptionChecker(hfcc); // so we can get result
4666         Collection<TableName> tables = getIncludedTables();
4667         Collection<Path> tableDirs = new ArrayList<Path>();
4668         Path rootdir = FSUtils.getRootDir(getConf());
4669         if (tables.size() > 0) {
4670           for (TableName t : tables) {
4671             tableDirs.add(FSUtils.getTableDir(rootdir, t));
4672           }
4673         } else {
4674           tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4675         }
4676         hfcc.checkTables(tableDirs);
4677         hfcc.report(errors);
4678       }
4679 
4680       // check and fix table integrity, region consistency.
4681       int code = onlineHbck();
4682       setRetCode(code);
4683       // If we have changed the HBase state it is better to run hbck again
4684       // to see if we haven't broken something else in the process.
4685       // We run it only once more because otherwise we can easily fall into
4686       // an infinite loop.
4687       if (shouldRerun()) {
4688         try {
4689           LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4690           Thread.sleep(sleepBeforeRerun);
4691         } catch (InterruptedException ie) {
4692           LOG.warn("Interrupted while sleeping");
4693           return this;
4694         }
4695         // Just report
4696         setFixAssignments(false);
4697         setFixMeta(false);
4698         setFixHdfsHoles(false);
4699         setFixHdfsOverlaps(false);
4700         setFixVersionFile(false);
4701         setFixTableOrphans(false);
4702         errors.resetErrors();
4703         code = onlineHbck();
4704         setRetCode(code);
4705       }
4706     } finally {
4707       IOUtils.closeQuietly(this);
4708     }
4709     return this;
4710   }
4711 
4712   /**
4713    * ls -r for debugging purposes
4714    */
4715   void debugLsr(Path p) throws IOException {
4716     debugLsr(getConf(), p, errors);
4717   }
4718 
4719   /**
4720    * ls -r for debugging purposes
4721    */
4722   public static void debugLsr(Configuration conf,
4723       Path p) throws IOException {
4724     debugLsr(conf, p, new PrintingErrorReporter());
4725   }
4726 
4727   /**
4728    * ls -r for debugging purposes
4729    */
4730   public static void debugLsr(Configuration conf,
4731       Path p, ErrorReporter errors) throws IOException {
4732     if (!LOG.isDebugEnabled() || p == null) {
4733       return;
4734     }
4735     FileSystem fs = p.getFileSystem(conf);
4736 
4737     if (!fs.exists(p)) {
4738       // nothing
4739       return;
4740     }
4741     errors.print(p.toString());
4742 
4743     if (fs.isFile(p)) {
4744       return;
4745     }
4746 
4747     if (fs.getFileStatus(p).isDirectory()) {
4748       FileStatus[] fss= fs.listStatus(p);
4749       for (FileStatus status : fss) {
4750         debugLsr(conf, status.getPath(), errors);
4751       }
4752     }
4753   }
4754 }