View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.Closeable;
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.io.InterruptedIOException;
24  import java.io.PrintWriter;
25  import java.io.StringWriter;
26  import java.net.InetAddress;
27  import java.net.URI;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collection;
31  import java.util.Collections;
32  import java.util.Comparator;
33  import java.util.HashMap;
34  import java.util.HashSet;
35  import java.util.Iterator;
36  import java.util.List;
37  import java.util.Map;
38  import java.util.Map.Entry;
39  import java.util.Set;
40  import java.util.SortedMap;
41  import java.util.SortedSet;
42  import java.util.TreeMap;
43  import java.util.TreeSet;
44  import java.util.concurrent.Callable;
45  import java.util.concurrent.ConcurrentSkipListMap;
46  import java.util.concurrent.ExecutionException;
47  import java.util.concurrent.ExecutorService;
48  import java.util.concurrent.Executors;
49  import java.util.concurrent.Future;
50  import java.util.concurrent.FutureTask;
51  import java.util.concurrent.ScheduledThreadPoolExecutor;
52  import java.util.concurrent.TimeUnit;
53  import java.util.concurrent.TimeoutException;
54  import java.util.concurrent.atomic.AtomicBoolean;
55  import java.util.concurrent.atomic.AtomicInteger;
56  
57  import com.google.common.base.Joiner;
58  import com.google.common.base.Preconditions;
59  import com.google.common.collect.ImmutableList;
60  import com.google.common.collect.Lists;
61  import com.google.common.collect.Multimap;
62  import com.google.common.collect.Ordering;
63  import com.google.common.collect.TreeMultimap;
64  import com.google.protobuf.ServiceException;
65  import org.apache.commons.io.IOUtils;
66  import org.apache.commons.lang.RandomStringUtils;
67  import org.apache.commons.lang.StringUtils;
68  import org.apache.commons.logging.Log;
69  import org.apache.commons.logging.LogFactory;
70  import org.apache.hadoop.conf.Configuration;
71  import org.apache.hadoop.conf.Configured;
72  import org.apache.hadoop.fs.FSDataOutputStream;
73  import org.apache.hadoop.fs.FileStatus;
74  import org.apache.hadoop.fs.FileSystem;
75  import org.apache.hadoop.fs.Path;
76  import org.apache.hadoop.fs.permission.FsAction;
77  import org.apache.hadoop.fs.permission.FsPermission;
78  import org.apache.hadoop.hbase.Abortable;
79  import org.apache.hadoop.hbase.Cell;
80  import org.apache.hadoop.hbase.CellUtil;
81  import org.apache.hadoop.hbase.ClusterStatus;
82  import org.apache.hadoop.hbase.HBaseConfiguration;
83  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
84  import org.apache.hadoop.hbase.HColumnDescriptor;
85  import org.apache.hadoop.hbase.HConstants;
86  import org.apache.hadoop.hbase.HRegionInfo;
87  import org.apache.hadoop.hbase.HRegionLocation;
88  import org.apache.hadoop.hbase.HTableDescriptor;
89  import org.apache.hadoop.hbase.KeyValue;
90  import org.apache.hadoop.hbase.MasterNotRunningException;
91  import org.apache.hadoop.hbase.MetaTableAccessor;
92  import org.apache.hadoop.hbase.RegionLocations;
93  import org.apache.hadoop.hbase.ServerName;
94  import org.apache.hadoop.hbase.TableDescriptor;
95  import org.apache.hadoop.hbase.TableName;
96  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
97  import org.apache.hadoop.hbase.classification.InterfaceAudience;
98  import org.apache.hadoop.hbase.classification.InterfaceStability;
99  import org.apache.hadoop.hbase.client.Admin;
100 import org.apache.hadoop.hbase.client.ClusterConnection;
101 import org.apache.hadoop.hbase.client.Connection;
102 import org.apache.hadoop.hbase.client.ConnectionFactory;
103 import org.apache.hadoop.hbase.client.Delete;
104 import org.apache.hadoop.hbase.client.Get;
105 import org.apache.hadoop.hbase.client.HConnection;
106 import org.apache.hadoop.hbase.client.Put;
107 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
108 import org.apache.hadoop.hbase.client.Result;
109 import org.apache.hadoop.hbase.client.RowMutations;
110 import org.apache.hadoop.hbase.client.Table;
111 import org.apache.hadoop.hbase.client.TableState;
112 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
113 import org.apache.hadoop.hbase.io.hfile.HFile;
114 import org.apache.hadoop.hbase.master.MasterFileSystem;
115 import org.apache.hadoop.hbase.master.RegionState;
116 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
117 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
118 import org.apache.hadoop.hbase.regionserver.HRegion;
119 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
120 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
121 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
122 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
123 import org.apache.hadoop.hbase.security.AccessDeniedException;
124 import org.apache.hadoop.hbase.security.UserProvider;
125 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
126 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
127 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
128 import org.apache.hadoop.hbase.util.hbck.ReplicationChecker;
129 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
130 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
131 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
132 import org.apache.hadoop.hbase.wal.WAL;
133 import org.apache.hadoop.hbase.wal.WALFactory;
134 import org.apache.hadoop.hbase.wal.WALSplitter;
135 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
136 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
137 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
138 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
139 import org.apache.hadoop.ipc.RemoteException;
140 import org.apache.hadoop.security.UserGroupInformation;
141 import org.apache.hadoop.util.ReflectionUtils;
142 import org.apache.hadoop.util.Tool;
143 import org.apache.hadoop.util.ToolRunner;
144 import org.apache.zookeeper.KeeperException;
145 
146 /**
147  * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
148  * table integrity problems in a corrupted HBase.
149  * <p>
150  * Region consistency checks verify that hbase:meta, region deployment on region
151  * servers and the state of data in HDFS (.regioninfo files) all are in
152  * accordance.
153  * <p>
154  * Table integrity checks verify that all possible row keys resolve to exactly
155  * one region of a table.  This means there are no individual degenerate
156  * or backwards regions; no holes between regions; and that there are no
157  * overlapping regions.
158  * <p>
159  * The general repair strategy works in two phases:
160  * <ol>
161  * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
162  * <li> Repair Region Consistency with hbase:meta and assignments
163  * </ol>
164  * <p>
165  * For table integrity repairs, the tables' region directories are scanned
166  * for .regioninfo files.  Each table's integrity is then verified.  If there
167  * are any orphan regions (regions with no .regioninfo files) or holes, new
168  * regions are fabricated.  Backwards regions are sidelined as well as empty
169  * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
170  * a new region is created and all data is merged into the new region.
171  * <p>
172  * Table integrity repairs deal solely with HDFS and could potentially be done
173  * offline -- the hbase region servers or master do not need to be running.
174  * This phase can eventually be used to completely reconstruct the hbase:meta table in
175  * an offline fashion.
176  * <p>
177  * Region consistency requires three conditions -- 1) valid .regioninfo file
178  * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
179  * and 3) a region is deployed only at the regionserver that was assigned to
180  * with proper state in the master.
181  * <p>
182  * Region consistency repairs require hbase to be online so that hbck can
183  * contact the HBase master and region servers.  The hbck#connect() method must
184  * first be called successfully.  Much of the region consistency information
185  * is transient and less risky to repair.
186  * <p>
187  * If hbck is run from the command line, there are a handful of arguments that
188  * can be used to limit the kinds of repairs hbck will do.  See the code in
189  * {@link #printUsageAndExit()} for more details.
190  */
191 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
192 @InterfaceStability.Evolving
193 public class HBaseFsck extends Configured implements Closeable {
194   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
195   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
196   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
197   private static boolean rsSupportsOffline = true;
198   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
199   private static final int DEFAULT_MAX_MERGE = 5;
200   private static final String TO_BE_LOADED = "to_be_loaded";
201   private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
202   private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5;
203   private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds
204   private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds
205   // We have to set the timeout value > HdfsConstants.LEASE_SOFTLIMIT_PERIOD.
206   // In HADOOP-2.6 and later, the Namenode proxy now created with custom RetryPolicy for
207   // AlreadyBeingCreatedException which is implies timeout on this operations up to
208   // HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds).
209   private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds
210 
211   /**********************
212    * Internal resources
213    **********************/
214   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
215   private ClusterStatus status;
216   private ClusterConnection connection;
217   private Admin admin;
218   private Table meta;
219   // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
220   protected ExecutorService executor;
221   private long startMillis = EnvironmentEdgeManager.currentTime();
222   private HFileCorruptionChecker hfcc;
223   private int retcode = 0;
224   private Path HBCK_LOCK_PATH;
225   private FSDataOutputStream hbckOutFd;
226   // This lock is to prevent cleanup of balancer resources twice between
227   // ShutdownHook and the main code. We cleanup only if the connect() is
228   // successful
229   private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
230 
231   /***********
232    * Options
233    ***********/
234   private static boolean details = false; // do we display the full report
235   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
236   private static boolean forceExclusive = false; // only this hbck can modify HBase
237   private static boolean disableBalancer = false; // disable load balancer to keep regions stable
238   private boolean fixAssignments = false; // fix assignment errors?
239   private boolean fixMeta = false; // fix meta errors?
240   private boolean checkHdfs = true; // load and check fs consistency?
241   private boolean fixHdfsHoles = false; // fix fs holes?
242   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
243   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
244   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
245   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
246   private boolean fixSplitParents = false; // fix lingering split parents
247   private boolean fixReferenceFiles = false; // fix lingering reference store file
248   private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
249   private boolean fixTableLocks = false; // fix table locks which are expired
250   private boolean fixReplication = false; // fix undeleted replication queues for removed peer
251   private boolean fixAny = false; // Set to true if any of the fix is required.
252 
253   // limit checking/fixes to listed tables, if empty attempt to check/fix all
254   // hbase:meta are always checked
255   private Set<TableName> tablesIncluded = new HashSet<TableName>();
256   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
257   // maximum number of overlapping regions to sideline
258   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE;
259   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
260   private Path sidelineDir = null;
261 
262   private boolean rerun = false; // if we tried to fix something, rerun hbck
263   private static boolean summary = false; // if we want to print less output
264   private boolean checkMetaOnly = false;
265   private boolean checkRegionBoundaries = false;
266   private boolean ignorePreCheckPermission = false; // if pre-check permission
267 
268   /*********
269    * State
270    *********/
271   final private ErrorReporter errors;
272   int fixes = 0;
273 
274   /**
275    * This map contains the state of all hbck items.  It maps from encoded region
276    * name to HbckInfo structure.  The information contained in HbckInfo is used
277    * to detect and correct consistency (hdfs/meta/deployment) problems.
278    */
279   private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
280   // Empty regioninfo qualifiers in hbase:meta
281   private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
282 
283   /**
284    * This map from Tablename -> TableInfo contains the structures necessary to
285    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
286    * to prevent dupes.
287    *
288    * If tablesIncluded is empty, this map contains all tables.
289    * Otherwise, it contains only meta tables and tables in tablesIncluded,
290    * unless checkMetaOnly is specified, in which case, it contains only
291    * the meta table
292    */
293   private SortedMap<TableName, TableInfo> tablesInfo =
294       new ConcurrentSkipListMap<TableName, TableInfo>();
295 
296   /**
297    * When initially looking at HDFS, we attempt to find any orphaned data.
298    */
299   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
300 
301   private Map<TableName, Set<String>> orphanTableDirs =
302       new HashMap<TableName, Set<String>>();
303   private Map<TableName, TableState> tableStates =
304       new HashMap<TableName, TableState>();
305   private final RetryCounterFactory lockFileRetryCounterFactory;
306 
307   private Map<TableName, Set<String>> skippedRegions = new HashMap<TableName, Set<String>>();
308 
309   /**
310    * Constructor
311    *
312    * @param conf Configuration object
313    * @throws MasterNotRunningException if the master is not running
314    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
315    */
316   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
317       ZooKeeperConnectionException, IOException, ClassNotFoundException {
318     this(conf, createThreadPool(conf));
319   }
320 
321   private static ExecutorService createThreadPool(Configuration conf) {
322     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
323     return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
324   }
325 
326   /**
327    * Constructor
328    *
329    * @param conf
330    *          Configuration object
331    * @throws MasterNotRunningException
332    *           if the master is not running
333    * @throws ZooKeeperConnectionException
334    *           if unable to connect to ZooKeeper
335    */
336   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
337       ZooKeeperConnectionException, IOException, ClassNotFoundException {
338     super(conf);
339     errors = getErrorReporter(getConf());
340     this.executor = exec;
341     lockFileRetryCounterFactory = new RetryCounterFactory(
342       getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
343       getConf().getInt(
344         "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),
345       getConf().getInt(
346         "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));
347   }
348 
349   private class FileLockCallable implements Callable<FSDataOutputStream> {
350     RetryCounter retryCounter;
351 
352     public FileLockCallable(RetryCounter retryCounter) {
353       this.retryCounter = retryCounter;
354     }
355     @Override
356     public FSDataOutputStream call() throws IOException {
357       try {
358         FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
359         FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
360             HConstants.DATA_FILE_UMASK_KEY);
361         Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
362         fs.mkdirs(tmpDir);
363         HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
364         final FSDataOutputStream out = createFileWithRetries(fs, HBCK_LOCK_PATH, defaultPerms);
365         out.writeBytes(InetAddress.getLocalHost().toString());
366         out.flush();
367         return out;
368       } catch(RemoteException e) {
369         if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
370           return null;
371         } else {
372           throw e;
373         }
374       }
375     }
376 
377     private FSDataOutputStream createFileWithRetries(final FileSystem fs,
378         final Path hbckLockFilePath, final FsPermission defaultPerms)
379         throws IOException {
380 
381       IOException exception = null;
382       do {
383         try {
384           return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false);
385         } catch (IOException ioe) {
386           LOG.info("Failed to create lock file " + hbckLockFilePath.getName()
387               + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of "
388               + retryCounter.getMaxAttempts());
389           LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(),
390               ioe);
391           try {
392             exception = ioe;
393             retryCounter.sleepUntilNextRetry();
394           } catch (InterruptedException ie) {
395             throw (InterruptedIOException) new InterruptedIOException(
396                 "Can't create lock file " + hbckLockFilePath.getName())
397             .initCause(ie);
398           }
399         }
400       } while (retryCounter.shouldRetry());
401 
402       throw exception;
403     }
404   }
405 
406   /**
407    * This method maintains a lock using a file. If the creation fails we return null
408    *
409    * @return FSDataOutputStream object corresponding to the newly opened lock file
410    * @throws IOException
411    */
412   private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
413     RetryCounter retryCounter = lockFileRetryCounterFactory.create();
414     FileLockCallable callable = new FileLockCallable(retryCounter);
415     ExecutorService executor = Executors.newFixedThreadPool(1);
416     FutureTask<FSDataOutputStream> futureTask = new FutureTask<FSDataOutputStream>(callable);
417     executor.execute(futureTask);
418     final int timeoutInSeconds = getConf().getInt(
419       "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT);
420     FSDataOutputStream stream = null;
421     try {
422       stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);
423     } catch (ExecutionException ee) {
424       LOG.warn("Encountered exception when opening lock file", ee);
425     } catch (InterruptedException ie) {
426       LOG.warn("Interrupted when opening lock file", ie);
427       Thread.currentThread().interrupt();
428     } catch (TimeoutException exception) {
429       // took too long to obtain lock
430       LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");
431       futureTask.cancel(true);
432     } finally {
433       executor.shutdownNow();
434     }
435     return stream;
436   }
437 
438   private void unlockHbck() {
439     if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) {
440       RetryCounter retryCounter = lockFileRetryCounterFactory.create();
441       do {
442         try {
443           IOUtils.closeQuietly(hbckOutFd);
444           FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()),
445               HBCK_LOCK_PATH, true);
446           LOG.info("Finishing hbck");
447           return;
448         } catch (IOException ioe) {
449           LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try="
450               + (retryCounter.getAttemptTimes() + 1) + " of "
451               + retryCounter.getMaxAttempts());
452           LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe);
453           try {
454             retryCounter.sleepUntilNextRetry();
455           } catch (InterruptedException ie) {
456             Thread.currentThread().interrupt();
457             LOG.warn("Interrupted while deleting lock file" +
458                 HBCK_LOCK_PATH);
459             return;
460           }
461         }
462       } while (retryCounter.shouldRetry());
463     }
464   }
465 
466   /**
467    * To repair region consistency, one must call connect() in order to repair
468    * online state.
469    */
470   public void connect() throws IOException {
471 
472     if (isExclusive()) {
473       // Grab the lock
474       hbckOutFd = checkAndMarkRunningHbck();
475       if (hbckOutFd == null) {
476         setRetCode(-1);
477         LOG.error("Another instance of hbck is fixing HBase, exiting this instance. " +
478             "[If you are sure no other instance is running, delete the lock file " +
479             HBCK_LOCK_PATH + " and rerun the tool]");
480         throw new IOException("Duplicate hbck - Abort");
481       }
482 
483       // Make sure to cleanup the lock
484       hbckLockCleanup.set(true);
485     }
486 
487 
488     // Add a shutdown hook to this thread, in case user tries to
489     // kill the hbck with a ctrl-c, we want to cleanup the lock so that
490     // it is available for further calls
491     Runtime.getRuntime().addShutdownHook(new Thread() {
492       @Override
493       public void run() {
494         IOUtils.closeQuietly(HBaseFsck.this);
495         unlockHbck();
496       }
497     });
498 
499     LOG.info("Launching hbck");
500 
501     connection = (ClusterConnection)ConnectionFactory.createConnection(getConf());
502     admin = connection.getAdmin();
503     meta = connection.getTable(TableName.META_TABLE_NAME);
504     status = admin.getClusterStatus();
505   }
506 
507   /**
508    * Get deployed regions according to the region servers.
509    */
510   private void loadDeployedRegions() throws IOException, InterruptedException {
511     // From the master, get a list of all known live region servers
512     Collection<ServerName> regionServers = status.getServers();
513     errors.print("Number of live region servers: " + regionServers.size());
514     if (details) {
515       for (ServerName rsinfo: regionServers) {
516         errors.print("  " + rsinfo.getServerName());
517       }
518     }
519 
520     // From the master, get a list of all dead region servers
521     Collection<ServerName> deadRegionServers = status.getDeadServerNames();
522     errors.print("Number of dead region servers: " + deadRegionServers.size());
523     if (details) {
524       for (ServerName name: deadRegionServers) {
525         errors.print("  " + name);
526       }
527     }
528 
529     // Print the current master name and state
530     errors.print("Master: " + status.getMaster());
531 
532     // Print the list of all backup masters
533     Collection<ServerName> backupMasters = status.getBackupMasters();
534     errors.print("Number of backup masters: " + backupMasters.size());
535     if (details) {
536       for (ServerName name: backupMasters) {
537         errors.print("  " + name);
538       }
539     }
540 
541     errors.print("Average load: " + status.getAverageLoad());
542     errors.print("Number of requests: " + status.getRequestsCount());
543     errors.print("Number of regions: " + status.getRegionsCount());
544 
545     Map<String, RegionState> rits = status.getRegionsInTransition();
546     errors.print("Number of regions in transition: " + rits.size());
547     if (details) {
548       for (RegionState state: rits.values()) {
549         errors.print("  " + state.toDescriptiveString());
550       }
551     }
552 
553     // Determine what's deployed
554     processRegionServers(regionServers);
555   }
556 
557   /**
558    * Clear the current state of hbck.
559    */
560   private void clearState() {
561     // Make sure regionInfo is empty before starting
562     fixes = 0;
563     regionInfoMap.clear();
564     emptyRegionInfoQualifiers.clear();
565     tableStates.clear();
566     errors.clear();
567     tablesInfo.clear();
568     orphanHdfsDirs.clear();
569     skippedRegions.clear();
570   }
571 
572   /**
573    * This repair method analyzes hbase data in hdfs and repairs it to satisfy
574    * the table integrity rules.  HBase doesn't need to be online for this
575    * operation to work.
576    */
577   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
578     // Initial pass to fix orphans.
579     if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
580         || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
581       LOG.info("Loading regioninfos HDFS");
582       // if nothing is happening this should always complete in two iterations.
583       int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
584       int curIter = 0;
585       do {
586         clearState(); // clears hbck state and reset fixes to 0 and.
587         // repair what's on HDFS
588         restoreHdfsIntegrity();
589         curIter++;// limit the number of iterations.
590       } while (fixes > 0 && curIter <= maxIterations);
591 
592       // Repairs should be done in the first iteration and verification in the second.
593       // If there are more than 2 passes, something funny has happened.
594       if (curIter > 2) {
595         if (curIter == maxIterations) {
596           LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
597               + "Tables integrity may not be fully repaired!");
598         } else {
599           LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
600         }
601       }
602     }
603   }
604 
605   /**
606    * This repair method requires the cluster to be online since it contacts
607    * region servers and the masters.  It makes each region's state in HDFS, in
608    * hbase:meta, and deployments consistent.
609    *
610    * @return If > 0 , number of errors detected, if &lt; 0 there was an unrecoverable
611    *     error.  If 0, we have a clean hbase.
612    */
613   public int onlineConsistencyRepair() throws IOException, KeeperException,
614     InterruptedException {
615     clearState();
616 
617     // get regions according to what is online on each RegionServer
618     loadDeployedRegions();
619     // check whether hbase:meta is deployed and online
620     recordMetaRegion();
621     // Check if hbase:meta is found only once and in the right place
622     if (!checkMetaRegion()) {
623       String errorMsg = "hbase:meta table is not consistent. ";
624       if (shouldFixAssignments()) {
625         errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
626       } else {
627         errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
628       }
629       errors.reportError(errorMsg + " Exiting...");
630       return -2;
631     }
632     // Not going with further consistency check for tables when hbase:meta itself is not consistent.
633     LOG.info("Loading regionsinfo from the hbase:meta table");
634     boolean success = loadMetaEntries();
635     if (!success) return -1;
636 
637     // Empty cells in hbase:meta?
638     reportEmptyMetaCells();
639 
640     // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
641     if (shouldFixEmptyMetaCells()) {
642       fixEmptyMetaCells();
643     }
644 
645     // get a list of all tables that have not changed recently.
646     if (!checkMetaOnly) {
647       reportTablesInFlux();
648     }
649 
650     // Get disabled tables states
651     loadTableStates();
652 
653     // load regiondirs and regioninfos from HDFS
654     if (shouldCheckHdfs()) {
655       LOG.info("Loading region directories from HDFS");
656       loadHdfsRegionDirs();
657       LOG.info("Loading region information from HDFS");
658       loadHdfsRegionInfos();
659     }
660 
661     // fix the orphan tables
662     fixOrphanTables();
663 
664     LOG.info("Checking and fixing region consistency");
665     // Check and fix consistency
666     checkAndFixConsistency();
667 
668     // Check integrity (does not fix)
669     checkIntegrity();
670     return errors.getErrorList().size();
671   }
672 
673   /**
674    * Contacts the master and prints out cluster-wide information
675    * @return 0 on success, non-zero on failure
676    */
677   public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
678     // print hbase server version
679     errors.print("Version: " + status.getHBaseVersion());
680     offlineHdfsIntegrityRepair();
681 
682     boolean oldBalancer = false;
683     if (shouldDisableBalancer()) {
684       oldBalancer = admin.setBalancerRunning(false, true);
685     }
686 
687     try {
688       onlineConsistencyRepair();
689     }
690     finally {
691       // Only restore the balancer if it was true when we started repairing and
692       // we actually disabled it. Otherwise, we might clobber another run of
693       // hbck that has just restored it.
694       if (shouldDisableBalancer() && oldBalancer) {
695         admin.setBalancerRunning(oldBalancer, false);
696       }
697     }
698 
699     if (checkRegionBoundaries) {
700       checkRegionBoundaries();
701     }
702 
703     offlineReferenceFileRepair();
704 
705     checkAndFixTableLocks();
706 
707     checkAndFixReplication();
708     
709     // Remove the hbck lock
710     unlockHbck();
711 
712     // Print table summary
713     printTableSummary(tablesInfo);
714     return errors.summarize();
715   }
716 
717   public static byte[] keyOnly (byte[] b) {
718     if (b == null)
719       return b;
720     int rowlength = Bytes.toShort(b, 0);
721     byte[] result = new byte[rowlength];
722     System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
723     return result;
724   }
725 
726   @Override
727   public void close() throws IOException {
728     try {
729       unlockHbck();
730     } catch (Exception io) {
731       LOG.warn(io);
732     } finally {
733       IOUtils.closeQuietly(admin);
734       IOUtils.closeQuietly(meta);
735       IOUtils.closeQuietly(connection);
736     }
737   }
738 
739   private static class RegionBoundariesInformation {
740     public byte [] regionName;
741     public byte [] metaFirstKey;
742     public byte [] metaLastKey;
743     public byte [] storesFirstKey;
744     public byte [] storesLastKey;
745     @Override
746     public String toString () {
747       return "regionName=" + Bytes.toStringBinary(regionName) +
748              "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
749              "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
750              "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
751              "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
752     }
753   }
754 
755   public void checkRegionBoundaries() {
756     try {
757       ByteArrayComparator comparator = new ByteArrayComparator();
758       List<HRegionInfo> regions = MetaTableAccessor.getAllRegions(connection, true);
759       final RegionBoundariesInformation currentRegionBoundariesInformation =
760           new RegionBoundariesInformation();
761       Path hbaseRoot = FSUtils.getRootDir(getConf());
762       for (HRegionInfo regionInfo : regions) {
763         Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
764         currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
765         // For each region, get the start and stop key from the META and compare them to the
766         // same information from the Stores.
767         Path path = new Path(tableDir, regionInfo.getEncodedName());
768         FileSystem fs = path.getFileSystem(getConf());
769         FileStatus[] files = fs.listStatus(path);
770         // For all the column families in this region...
771         byte[] storeFirstKey = null;
772         byte[] storeLastKey = null;
773         for (FileStatus file : files) {
774           String fileName = file.getPath().toString();
775           fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
776           if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
777             FileStatus[] storeFiles = fs.listStatus(file.getPath());
778             // For all the stores in this column family.
779             for (FileStatus storeFile : storeFiles) {
780               HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
781                   getConf()), getConf());
782               if ((reader.getFirstKey() != null)
783                   && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
784                       ((KeyValue.KeyOnlyKeyValue) reader.getFirstKey()).getKey()) > 0))) {
785                 storeFirstKey = ((KeyValue.KeyOnlyKeyValue)reader.getFirstKey()).getKey();
786               }
787               if ((reader.getLastKey() != null)
788                   && ((storeLastKey == null) || (comparator.compare(storeLastKey,
789                       ((KeyValue.KeyOnlyKeyValue)reader.getLastKey()).getKey())) < 0)) {
790                 storeLastKey = ((KeyValue.KeyOnlyKeyValue)reader.getLastKey()).getKey();
791               }
792               reader.close();
793             }
794           }
795         }
796         currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
797         currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
798         currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
799         currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
800         if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
801           currentRegionBoundariesInformation.metaFirstKey = null;
802         if (currentRegionBoundariesInformation.metaLastKey.length == 0)
803           currentRegionBoundariesInformation.metaLastKey = null;
804 
805         // For a region to be correct, we need the META start key to be smaller or equal to the
806         // smallest start key from all the stores, and the start key from the next META entry to
807         // be bigger than the last key from all the current stores. First region start key is null;
808         // Last region end key is null; some regions can be empty and not have any store.
809 
810         boolean valid = true;
811         // Checking start key.
812         if ((currentRegionBoundariesInformation.storesFirstKey != null)
813             && (currentRegionBoundariesInformation.metaFirstKey != null)) {
814           valid = valid
815               && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
816                 currentRegionBoundariesInformation.metaFirstKey) >= 0;
817         }
818         // Checking stop key.
819         if ((currentRegionBoundariesInformation.storesLastKey != null)
820             && (currentRegionBoundariesInformation.metaLastKey != null)) {
821           valid = valid
822               && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
823                 currentRegionBoundariesInformation.metaLastKey) < 0;
824         }
825         if (!valid) {
826           errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
827             tablesInfo.get(regionInfo.getTable()));
828           LOG.warn("Region's boundaries not alligned between stores and META for:");
829           LOG.warn(currentRegionBoundariesInformation);
830         }
831       }
832     } catch (IOException e) {
833       LOG.error(e);
834     }
835   }
836 
837   /**
838    * Iterates through the list of all orphan/invalid regiondirs.
839    */
840   private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
841     for (HbckInfo hi : orphanHdfsDirs) {
842       LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
843       adoptHdfsOrphan(hi);
844     }
845   }
846 
847   /**
848    * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
849    * these orphans by creating a new region, and moving the column families,
850    * recovered edits, WALs, into the new region dir.  We determine the region
851    * startkey and endkeys by looking at all of the hfiles inside the column
852    * families to identify the min and max keys. The resulting region will
853    * likely violate table integrity but will be dealt with by merging
854    * overlapping regions.
855    */
856   @SuppressWarnings("deprecation")
857   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
858     Path p = hi.getHdfsRegionDir();
859     FileSystem fs = p.getFileSystem(getConf());
860     FileStatus[] dirs = fs.listStatus(p);
861     if (dirs == null) {
862       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
863           p + ". This dir could probably be deleted.");
864       return ;
865     }
866 
867     TableName tableName = hi.getTableName();
868     TableInfo tableInfo = tablesInfo.get(tableName);
869     Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
870     HTableDescriptor template = tableInfo.getHTD();
871 
872     // find min and max key values
873     Pair<byte[],byte[]> orphanRegionRange = null;
874     for (FileStatus cf : dirs) {
875       String cfName= cf.getPath().getName();
876       // TODO Figure out what the special dirs are
877       if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
878 
879       FileStatus[] hfiles = fs.listStatus(cf.getPath());
880       for (FileStatus hfile : hfiles) {
881         byte[] start, end;
882         HFile.Reader hf = null;
883         try {
884           CacheConfig cacheConf = new CacheConfig(getConf());
885           hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
886           hf.loadFileInfo();
887           Cell startKv = hf.getFirstKey();
888           start = CellUtil.cloneRow(startKv);
889           Cell endKv = hf.getLastKey();
890           end = CellUtil.cloneRow(endKv);
891         } catch (IOException ioe) {
892           LOG.warn("Problem reading orphan file " + hfile + ", skipping");
893           continue;
894         } catch (NullPointerException ioe) {
895           LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
896           continue;
897         } finally {
898           if (hf != null) {
899             hf.close();
900           }
901         }
902 
903         // expand the range to include the range of all hfiles
904         if (orphanRegionRange == null) {
905           // first range
906           orphanRegionRange = new Pair<byte[], byte[]>(start, end);
907         } else {
908           // TODO add test
909 
910           // expand range only if the hfile is wider.
911           if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
912             orphanRegionRange.setFirst(start);
913           }
914           if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
915             orphanRegionRange.setSecond(end);
916           }
917         }
918       }
919     }
920     if (orphanRegionRange == null) {
921       LOG.warn("No data in dir " + p + ", sidelining data");
922       fixes++;
923       sidelineRegionDir(fs, hi);
924       return;
925     }
926     LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
927         Bytes.toString(orphanRegionRange.getSecond()) + ")");
928 
929     // create new region on hdfs. move data into place.
930     HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(),
931         Bytes.add(orphanRegionRange.getSecond(), new byte[1]));
932     LOG.info("Creating new region : " + hri);
933     HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
934     Path target = region.getRegionFileSystem().getRegionDir();
935 
936     // rename all the data to new region
937     mergeRegionDirs(target, hi);
938     fixes++;
939   }
940 
941   /**
942    * This method determines if there are table integrity errors in HDFS.  If
943    * there are errors and the appropriate "fix" options are enabled, the method
944    * will first correct orphan regions making them into legit regiondirs, and
945    * then reload to merge potentially overlapping regions.
946    *
947    * @return number of table integrity errors found
948    */
949   private int restoreHdfsIntegrity() throws IOException, InterruptedException {
950     // Determine what's on HDFS
951     LOG.info("Loading HBase regioninfo from HDFS...");
952     loadHdfsRegionDirs(); // populating regioninfo table.
953 
954     int errs = errors.getErrorList().size();
955     // First time just get suggestions.
956     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
957     checkHdfsIntegrity(false, false);
958 
959     if (errors.getErrorList().size() == errs) {
960       LOG.info("No integrity errors.  We are done with this phase. Glorious.");
961       return 0;
962     }
963 
964     if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
965       adoptHdfsOrphans(orphanHdfsDirs);
966       // TODO optimize by incrementally adding instead of reloading.
967     }
968 
969     // Make sure there are no holes now.
970     if (shouldFixHdfsHoles()) {
971       clearState(); // this also resets # fixes.
972       loadHdfsRegionDirs();
973       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
974       tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
975     }
976 
977     // Now we fix overlaps
978     if (shouldFixHdfsOverlaps()) {
979       // second pass we fix overlaps.
980       clearState(); // this also resets # fixes.
981       loadHdfsRegionDirs();
982       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
983       tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
984     }
985 
986     return errors.getErrorList().size();
987   }
988 
989   /**
990    * Scan all the store file names to find any lingering reference files,
991    * which refer to some none-exiting files. If "fix" option is enabled,
992    * any lingering reference file will be sidelined if found.
993    * <p>
994    * Lingering reference file prevents a region from opening. It has to
995    * be fixed before a cluster can start properly.
996    */
997   private void offlineReferenceFileRepair() throws IOException {
998     Configuration conf = getConf();
999     Path hbaseRoot = FSUtils.getRootDir(conf);
1000     FileSystem fs = hbaseRoot.getFileSystem(conf);
1001     LOG.info("Computing mapping of all store files");
1002     Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot, errors);
1003     errors.print("");
1004     LOG.info("Validating mapping using HDFS state");
1005     for (Path path: allFiles.values()) {
1006       boolean isReference = false;
1007       try {
1008         isReference = StoreFileInfo.isReference(path);
1009       } catch (Throwable t) {
1010         // Ignore. Some files may not be store files at all.
1011         // For example, files under .oldlogs folder in hbase:meta
1012         // Warning message is already logged by
1013         // StoreFile#isReference.
1014       }
1015       if (!isReference) continue;
1016 
1017       Path referredToFile = StoreFileInfo.getReferredToFile(path);
1018       if (fs.exists(referredToFile)) continue;  // good, expected
1019 
1020       // Found a lingering reference file
1021       errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
1022         "Found lingering reference file " + path);
1023       if (!shouldFixReferenceFiles()) continue;
1024 
1025       // Now, trying to fix it since requested
1026       boolean success = false;
1027       String pathStr = path.toString();
1028 
1029       // A reference file path should be like
1030       // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
1031       // Up 5 directories to get the root folder.
1032       // So the file will be sidelined to a similar folder structure.
1033       int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
1034       for (int i = 0; index > 0 && i < 5; i++) {
1035         index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
1036       }
1037       if (index > 0) {
1038         Path rootDir = getSidelineDir();
1039         Path dst = new Path(rootDir, pathStr.substring(index + 1));
1040         fs.mkdirs(dst.getParent());
1041         LOG.info("Trying to sildeline reference file "
1042           + path + " to " + dst);
1043         setShouldRerun();
1044 
1045         success = fs.rename(path, dst);
1046       }
1047       if (!success) {
1048         LOG.error("Failed to sideline reference file " + path);
1049       }
1050     }
1051   }
1052 
1053   /**
1054    * TODO -- need to add tests for this.
1055    */
1056   private void reportEmptyMetaCells() {
1057     errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
1058       emptyRegionInfoQualifiers.size());
1059     if (details) {
1060       for (Result r: emptyRegionInfoQualifiers) {
1061         errors.print("  " + r);
1062       }
1063     }
1064   }
1065 
1066   /**
1067    * TODO -- need to add tests for this.
1068    */
1069   private void reportTablesInFlux() {
1070     AtomicInteger numSkipped = new AtomicInteger(0);
1071     HTableDescriptor[] allTables = getTables(numSkipped);
1072     errors.print("Number of Tables: " + allTables.length);
1073     if (details) {
1074       if (numSkipped.get() > 0) {
1075         errors.detail("Number of Tables in flux: " + numSkipped.get());
1076       }
1077       for (HTableDescriptor td : allTables) {
1078         errors.detail("  Table: " + td.getTableName() + "\t" +
1079                            (td.isReadOnly() ? "ro" : "rw") + "\t" +
1080                             (td.isMetaRegion() ? "META" : "    ") + "\t" +
1081                            " families: " + td.getFamilies().size());
1082       }
1083     }
1084   }
1085 
1086   public ErrorReporter getErrors() {
1087     return errors;
1088   }
1089 
1090   /**
1091    * Read the .regioninfo file from the file system.  If there is no
1092    * .regioninfo, add it to the orphan hdfs region list.
1093    */
1094   private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
1095     Path regionDir = hbi.getHdfsRegionDir();
1096     if (regionDir == null) {
1097       LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
1098       return;
1099     }
1100 
1101     if (hbi.hdfsEntry.hri != null) {
1102       // already loaded data
1103       return;
1104     }
1105 
1106     FileSystem fs = FileSystem.get(getConf());
1107     HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
1108     LOG.debug("HRegionInfo read: " + hri.toString());
1109     hbi.hdfsEntry.hri = hri;
1110   }
1111 
1112   /**
1113    * Exception thrown when a integrity repair operation fails in an
1114    * unresolvable way.
1115    */
1116   public static class RegionRepairException extends IOException {
1117     private static final long serialVersionUID = 1L;
1118     final IOException ioe;
1119     public RegionRepairException(String s, IOException ioe) {
1120       super(s);
1121       this.ioe = ioe;
1122     }
1123   }
1124 
1125   /**
1126    * Populate hbi's from regionInfos loaded from file system.
1127    */
1128   private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
1129       throws IOException, InterruptedException {
1130     tablesInfo.clear(); // regenerating the data
1131     // generate region split structure
1132     Collection<HbckInfo> hbckInfos = regionInfoMap.values();
1133 
1134     // Parallelized read of .regioninfo files.
1135     List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
1136     List<Future<Void>> hbiFutures;
1137 
1138     for (HbckInfo hbi : hbckInfos) {
1139       WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1140       hbis.add(work);
1141     }
1142 
1143     // Submit and wait for completion
1144     hbiFutures = executor.invokeAll(hbis);
1145 
1146     for(int i=0; i<hbiFutures.size(); i++) {
1147       WorkItemHdfsRegionInfo work = hbis.get(i);
1148       Future<Void> f = hbiFutures.get(i);
1149       try {
1150         f.get();
1151       } catch(ExecutionException e) {
1152         LOG.warn("Failed to read .regioninfo file for region " +
1153               work.hbi.getRegionNameAsString(), e.getCause());
1154       }
1155     }
1156 
1157     Path hbaseRoot = FSUtils.getRootDir(getConf());
1158     FileSystem fs = hbaseRoot.getFileSystem(getConf());
1159     // serialized table info gathering.
1160     for (HbckInfo hbi: hbckInfos) {
1161 
1162       if (hbi.getHdfsHRI() == null) {
1163         // was an orphan
1164         continue;
1165       }
1166 
1167 
1168       // get table name from hdfs, populate various HBaseFsck tables.
1169       TableName tableName = hbi.getTableName();
1170       if (tableName == null) {
1171         // There was an entry in hbase:meta not in the HDFS?
1172         LOG.warn("tableName was null for: " + hbi);
1173         continue;
1174       }
1175 
1176       TableInfo modTInfo = tablesInfo.get(tableName);
1177       if (modTInfo == null) {
1178         // only executed once per table.
1179         modTInfo = new TableInfo(tableName);
1180         tablesInfo.put(tableName, modTInfo);
1181         try {
1182           TableDescriptor htd =
1183               FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1184           modTInfo.htds.add(htd.getHTableDescriptor());
1185         } catch (IOException ioe) {
1186           if (!orphanTableDirs.containsKey(tableName)) {
1187             LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1188             //should only report once for each table
1189             errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1190                 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1191             Set<String> columns = new HashSet<String>();
1192             orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1193           }
1194         }
1195       }
1196       if (!hbi.isSkipChecks()) {
1197         modTInfo.addRegionInfo(hbi);
1198       }
1199     }
1200 
1201     loadTableInfosForTablesWithNoRegion();
1202     errors.print("");
1203 
1204     return tablesInfo;
1205   }
1206 
1207   /**
1208    * To get the column family list according to the column family dirs
1209    * @param columns
1210    * @param hbi
1211    * @return a set of column families
1212    * @throws IOException
1213    */
1214   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
1215     Path regionDir = hbi.getHdfsRegionDir();
1216     FileSystem fs = regionDir.getFileSystem(getConf());
1217     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1218     for (FileStatus subdir : subDirs) {
1219       String columnfamily = subdir.getPath().getName();
1220       columns.add(columnfamily);
1221     }
1222     return columns;
1223   }
1224 
1225   /**
1226    * To fabricate a .tableinfo file with following contents<br>
1227    * 1. the correct tablename <br>
1228    * 2. the correct colfamily list<br>
1229    * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1230    * @throws IOException
1231    */
1232   private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1233       Set<String> columns) throws IOException {
1234     if (columns ==null || columns.isEmpty()) return false;
1235     HTableDescriptor htd = new HTableDescriptor(tableName);
1236     for (String columnfamimly : columns) {
1237       htd.addFamily(new HColumnDescriptor(columnfamimly));
1238     }
1239     fstd.createTableDescriptor(new TableDescriptor(htd), true);
1240     return true;
1241   }
1242 
1243   /**
1244    * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1245    * @throws IOException
1246    */
1247   public void fixEmptyMetaCells() throws IOException {
1248     if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1249       LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1250       for (Result region : emptyRegionInfoQualifiers) {
1251         deleteMetaRegion(region.getRow());
1252         errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1253       }
1254       emptyRegionInfoQualifiers.clear();
1255     }
1256   }
1257 
1258   /**
1259    * To fix orphan table by creating a .tableinfo file under tableDir <br>
1260    * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1261    * 2. else create a default .tableinfo file with following items<br>
1262    * &nbsp;2.1 the correct tablename <br>
1263    * &nbsp;2.2 the correct colfamily list<br>
1264    * &nbsp;2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1265    * @throws IOException
1266    */
1267   public void fixOrphanTables() throws IOException {
1268     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1269 
1270       List<TableName> tmpList = new ArrayList<TableName>();
1271       tmpList.addAll(orphanTableDirs.keySet());
1272       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1273       Iterator<Entry<TableName, Set<String>>> iter =
1274           orphanTableDirs.entrySet().iterator();
1275       int j = 0;
1276       int numFailedCase = 0;
1277       FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1278       while (iter.hasNext()) {
1279         Entry<TableName, Set<String>> entry =
1280             iter.next();
1281         TableName tableName = entry.getKey();
1282         LOG.info("Trying to fix orphan table error: " + tableName);
1283         if (j < htds.length) {
1284           if (tableName.equals(htds[j].getTableName())) {
1285             HTableDescriptor htd = htds[j];
1286             LOG.info("fixing orphan table: " + tableName + " from cache");
1287             fstd.createTableDescriptor(new TableDescriptor(htd), true);
1288             j++;
1289             iter.remove();
1290           }
1291         } else {
1292           if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1293             LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1294             LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1295             iter.remove();
1296           } else {
1297             LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1298             numFailedCase++;
1299           }
1300         }
1301         fixes++;
1302       }
1303 
1304       if (orphanTableDirs.isEmpty()) {
1305         // all orphanTableDirs are luckily recovered
1306         // re-run doFsck after recovering the .tableinfo file
1307         setShouldRerun();
1308         LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1309       } else if (numFailedCase > 0) {
1310         LOG.error("Failed to fix " + numFailedCase
1311             + " OrphanTables with default .tableinfo files");
1312       }
1313 
1314     }
1315     //cleanup the list
1316     orphanTableDirs.clear();
1317 
1318   }
1319 
1320   /**
1321    * This borrows code from MasterFileSystem.bootstrap(). Explicitly creates it's own WAL, so be
1322    * sure to close it as well as the region when you're finished.
1323    *
1324    * @return an open hbase:meta HRegion
1325    */
1326   private HRegion createNewMeta() throws IOException {
1327     Path rootdir = FSUtils.getRootDir(getConf());
1328     Configuration c = getConf();
1329     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1330     HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
1331     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false);
1332     // The WAL subsystem will use the default rootDir rather than the passed in rootDir
1333     // unless I pass along via the conf.
1334     Configuration confForWAL = new Configuration(c);
1335     confForWAL.set(HConstants.HBASE_DIR, rootdir.toString());
1336     WAL wal = (new WALFactory(confForWAL,
1337         Collections.<WALActionsListener>singletonList(new MetricsWAL()),
1338         "hbck-meta-recovery-" + RandomStringUtils.randomNumeric(8))).
1339         getWAL(metaHRI.getEncodedNameAsBytes(), metaHRI.getTable().getNamespace());
1340     HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor, wal);
1341     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
1342     return meta;
1343   }
1344 
1345   /**
1346    * Generate set of puts to add to new meta.  This expects the tables to be
1347    * clean with no overlaps or holes.  If there are any problems it returns null.
1348    *
1349    * @return An array list of puts to do in bulk, null if tables have problems
1350    */
1351   private ArrayList<Put> generatePuts(
1352       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1353     ArrayList<Put> puts = new ArrayList<Put>();
1354     boolean hasProblems = false;
1355     for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1356       TableName name = e.getKey();
1357 
1358       // skip "hbase:meta"
1359       if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1360         continue;
1361       }
1362 
1363       TableInfo ti = e.getValue();
1364       puts.add(MetaTableAccessor
1365           .makePutFromTableState(new TableState(ti.tableName, TableState.State.ENABLED)));
1366       for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1367           .entrySet()) {
1368         Collection<HbckInfo> his = spl.getValue();
1369         int sz = his.size();
1370         if (sz != 1) {
1371           // problem
1372           LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1373               + " had " +  sz + " regions instead of exactly 1." );
1374           hasProblems = true;
1375           continue;
1376         }
1377 
1378         // add the row directly to meta.
1379         HbckInfo hi = his.iterator().next();
1380         HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
1381         Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1382         puts.add(p);
1383       }
1384     }
1385     return hasProblems ? null : puts;
1386   }
1387 
1388   /**
1389    * Suggest fixes for each table
1390    */
1391   private void suggestFixes(
1392       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1393     logParallelMerge();
1394     for (TableInfo tInfo : tablesInfo.values()) {
1395       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1396       tInfo.checkRegionChain(handler);
1397     }
1398   }
1399 
1400   /**
1401    * Rebuilds meta from information in hdfs/fs.  Depends on configuration settings passed into
1402    * hbck constructor to point to a particular fs/dir. Assumes HBase is OFFLINE.
1403    *
1404    * @param fix flag that determines if method should attempt to fix holes
1405    * @return true if successful, false if attempt failed.
1406    */
1407   public boolean rebuildMeta(boolean fix) throws IOException,
1408       InterruptedException {
1409 
1410     // TODO check to make sure hbase is offline. (or at least the table
1411     // currently being worked on is off line)
1412 
1413     // Determine what's on HDFS
1414     LOG.info("Loading HBase regioninfo from HDFS...");
1415     loadHdfsRegionDirs(); // populating regioninfo table.
1416 
1417     int errs = errors.getErrorList().size();
1418     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1419     checkHdfsIntegrity(false, false);
1420 
1421     // make sure ok.
1422     if (errors.getErrorList().size() != errs) {
1423       // While in error state, iterate until no more fixes possible
1424       while(true) {
1425         fixes = 0;
1426         suggestFixes(tablesInfo);
1427         errors.clear();
1428         loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1429         checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1430 
1431         int errCount = errors.getErrorList().size();
1432 
1433         if (fixes == 0) {
1434           if (errCount > 0) {
1435             return false; // failed to fix problems.
1436           } else {
1437             break; // no fixes and no problems? drop out and fix stuff!
1438           }
1439         }
1440       }
1441     }
1442 
1443     // we can rebuild, move old meta out of the way and start
1444     LOG.info("HDFS regioninfo's seems good.  Sidelining old hbase:meta");
1445     Path backupDir = sidelineOldMeta();
1446 
1447     LOG.info("Creating new hbase:meta");
1448     HRegion meta = createNewMeta();
1449 
1450     // populate meta
1451     List<Put> puts = generatePuts(tablesInfo);
1452     if (puts == null) {
1453       LOG.fatal("Problem encountered when creating new hbase:meta entries.  " +
1454         "You may need to restore the previously sidelined hbase:meta");
1455       return false;
1456     }
1457     meta.batchMutate(puts.toArray(new Put[puts.size()]), HConstants.NO_NONCE, HConstants.NO_NONCE);
1458     meta.close();
1459     if (meta.getWAL() != null) {
1460       meta.getWAL().close();
1461     }
1462     LOG.info("Success! hbase:meta table rebuilt.");
1463     LOG.info("Old hbase:meta is moved into " + backupDir);
1464     return true;
1465   }
1466 
1467   /**
1468    * Log an appropriate message about whether or not overlapping merges are computed in parallel.
1469    */
1470   private void logParallelMerge() {
1471     if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
1472       LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
1473           " false to run serially.");
1474     } else {
1475       LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
1476           " true to run in parallel.");
1477     }
1478   }
1479 
1480   private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1481       boolean fixOverlaps) throws IOException {
1482     LOG.info("Checking HBase region split map from HDFS data...");
1483     logParallelMerge();
1484     for (TableInfo tInfo : tablesInfo.values()) {
1485       TableIntegrityErrorHandler handler;
1486       if (fixHoles || fixOverlaps) {
1487         handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1488           fixHoles, fixOverlaps);
1489       } else {
1490         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1491       }
1492       if (!tInfo.checkRegionChain(handler)) {
1493         // should dump info as well.
1494         errors.report("Found inconsistency in table " + tInfo.getName());
1495       }
1496     }
1497     return tablesInfo;
1498   }
1499 
1500   private Path getSidelineDir() throws IOException {
1501     if (sidelineDir == null) {
1502       Path hbaseDir = FSUtils.getRootDir(getConf());
1503       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1504       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1505           + startMillis);
1506     }
1507     return sidelineDir;
1508   }
1509 
1510   /**
1511    * Sideline a region dir (instead of deleting it)
1512    */
1513   Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1514     return sidelineRegionDir(fs, null, hi);
1515   }
1516 
1517   /**
1518    * Sideline a region dir (instead of deleting it)
1519    *
1520    * @param parentDir if specified, the region will be sidelined to folder like
1521    *     {@literal .../parentDir/<table name>/<region name>}. The purpose is to group together
1522    *     similar regions sidelined, for example, those regions should be bulk loaded back later
1523    *     on. If NULL, it is ignored.
1524    */
1525   Path sidelineRegionDir(FileSystem fs,
1526       String parentDir, HbckInfo hi) throws IOException {
1527     TableName tableName = hi.getTableName();
1528     Path regionDir = hi.getHdfsRegionDir();
1529 
1530     if (!fs.exists(regionDir)) {
1531       LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1532       return null;
1533     }
1534 
1535     Path rootDir = getSidelineDir();
1536     if (parentDir != null) {
1537       rootDir = new Path(rootDir, parentDir);
1538     }
1539     Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1540     Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1541     fs.mkdirs(sidelineRegionDir);
1542     boolean success = false;
1543     FileStatus[] cfs =  fs.listStatus(regionDir);
1544     if (cfs == null) {
1545       LOG.info("Region dir is empty: " + regionDir);
1546     } else {
1547       for (FileStatus cf : cfs) {
1548         Path src = cf.getPath();
1549         Path dst =  new Path(sidelineRegionDir, src.getName());
1550         if (fs.isFile(src)) {
1551           // simple file
1552           success = fs.rename(src, dst);
1553           if (!success) {
1554             String msg = "Unable to rename file " + src +  " to " + dst;
1555             LOG.error(msg);
1556             throw new IOException(msg);
1557           }
1558           continue;
1559         }
1560 
1561         // is a directory.
1562         fs.mkdirs(dst);
1563 
1564         LOG.info("Sidelining files from " + src + " into containing region " + dst);
1565         // FileSystem.rename is inconsistent with directories -- if the
1566         // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1567         // it moves the src into the dst dir resulting in (foo/a/b).  If
1568         // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1569         FileStatus[] hfiles = fs.listStatus(src);
1570         if (hfiles != null && hfiles.length > 0) {
1571           for (FileStatus hfile : hfiles) {
1572             success = fs.rename(hfile.getPath(), dst);
1573             if (!success) {
1574               String msg = "Unable to rename file " + src +  " to " + dst;
1575               LOG.error(msg);
1576               throw new IOException(msg);
1577             }
1578           }
1579         }
1580         LOG.debug("Sideline directory contents:");
1581         debugLsr(sidelineRegionDir);
1582       }
1583     }
1584 
1585     LOG.info("Removing old region dir: " + regionDir);
1586     success = fs.delete(regionDir, true);
1587     if (!success) {
1588       String msg = "Unable to delete dir " + regionDir;
1589       LOG.error(msg);
1590       throw new IOException(msg);
1591     }
1592     return sidelineRegionDir;
1593   }
1594 
1595   /**
1596    * Side line an entire table.
1597    */
1598   void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1599       Path backupHbaseDir) throws IOException {
1600     Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1601     if (fs.exists(tableDir)) {
1602       Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1603       fs.mkdirs(backupTableDir.getParent());
1604       boolean success = fs.rename(tableDir, backupTableDir);
1605       if (!success) {
1606         throw new IOException("Failed to move  " + tableName + " from "
1607             +  tableDir + " to " + backupTableDir);
1608       }
1609     } else {
1610       LOG.info("No previous " + tableName +  " exists.  Continuing.");
1611     }
1612   }
1613 
1614   /**
1615    * @return Path to backup of original directory
1616    */
1617   Path sidelineOldMeta() throws IOException {
1618     // put current hbase:meta aside.
1619     Path hbaseDir = FSUtils.getRootDir(getConf());
1620     FileSystem fs = hbaseDir.getFileSystem(getConf());
1621     Path backupDir = getSidelineDir();
1622     fs.mkdirs(backupDir);
1623 
1624     try {
1625       sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1626     } catch (IOException e) {
1627         LOG.fatal("... failed to sideline meta. Currently in inconsistent state.  To restore "
1628             + "try to rename hbase:meta in " + backupDir.getName() + " to "
1629             + hbaseDir.getName() + ".", e);
1630       throw e; // throw original exception
1631     }
1632     return backupDir;
1633   }
1634 
1635   /**
1636    * Load the list of disabled tables in ZK into local set.
1637    * @throws ZooKeeperConnectionException
1638    * @throws IOException
1639    */
1640   private void loadTableStates()
1641   throws IOException {
1642     tableStates = MetaTableAccessor.getTableStates(connection);
1643   }
1644 
1645   /**
1646    * Check if the specified region's table is disabled.
1647    * @param tableName table to check status of
1648    */
1649   private boolean isTableDisabled(TableName tableName) {
1650     return tableStates.containsKey(tableName)
1651         && tableStates.get(tableName)
1652         .inStates(TableState.State.DISABLED, TableState.State.DISABLING);
1653   }
1654 
1655   /**
1656    * Scan HDFS for all regions, recording their information into
1657    * regionInfoMap
1658    */
1659   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1660     Path rootDir = FSUtils.getRootDir(getConf());
1661     FileSystem fs = rootDir.getFileSystem(getConf());
1662 
1663     // list all tables from HDFS
1664     List<FileStatus> tableDirs = Lists.newArrayList();
1665 
1666     boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1667 
1668     List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1669     for (Path path : paths) {
1670       TableName tableName = FSUtils.getTableName(path);
1671        if ((!checkMetaOnly &&
1672            isTableIncluded(tableName)) ||
1673            tableName.equals(TableName.META_TABLE_NAME)) {
1674          tableDirs.add(fs.getFileStatus(path));
1675        }
1676     }
1677 
1678     // verify that version file exists
1679     if (!foundVersionFile) {
1680       errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1681           "Version file does not exist in root dir " + rootDir);
1682       if (shouldFixVersionFile()) {
1683         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1684             + " file.");
1685         setShouldRerun();
1686         FSUtils.setVersion(fs, rootDir, getConf().getInt(
1687             HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1688             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1689             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1690       }
1691     }
1692 
1693     // level 1:  <HBASE_DIR>/*
1694     List<WorkItemHdfsDir> dirs = new ArrayList<WorkItemHdfsDir>(tableDirs.size());
1695     List<Future<Void>> dirsFutures;
1696 
1697     for (FileStatus tableDir : tableDirs) {
1698       LOG.debug("Loading region dirs from " +tableDir.getPath());
1699       dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir));
1700     }
1701 
1702     // Invoke and wait for Callables to complete
1703     dirsFutures = executor.invokeAll(dirs);
1704 
1705     for(Future<Void> f: dirsFutures) {
1706       try {
1707         f.get();
1708       } catch(ExecutionException e) {
1709         LOG.warn("Could not load region dir " , e.getCause());
1710       }
1711     }
1712     errors.print("");
1713   }
1714 
1715   /**
1716    * Record the location of the hbase:meta region as found in ZooKeeper.
1717    */
1718   private boolean recordMetaRegion() throws IOException {
1719     RegionLocations rl = ((ClusterConnection)connection).locateRegion(TableName.META_TABLE_NAME,
1720         HConstants.EMPTY_START_ROW, false, false);
1721     if (rl == null) {
1722       errors.reportError(ERROR_CODE.NULL_META_REGION,
1723           "META region was not found in Zookeeper");
1724       return false;
1725     }
1726     for (HRegionLocation metaLocation : rl.getRegionLocations()) {
1727       // Check if Meta region is valid and existing
1728       if (metaLocation == null ) {
1729         errors.reportError(ERROR_CODE.NULL_META_REGION,
1730             "META region location is null");
1731         return false;
1732       }
1733       if (metaLocation.getRegionInfo() == null) {
1734         errors.reportError(ERROR_CODE.NULL_META_REGION,
1735             "META location regionInfo is null");
1736         return false;
1737       }
1738       if (metaLocation.getHostname() == null) {
1739         errors.reportError(ERROR_CODE.NULL_META_REGION,
1740             "META location hostName is null");
1741         return false;
1742       }
1743       ServerName sn = metaLocation.getServerName();
1744       MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, EnvironmentEdgeManager.currentTime());
1745       HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1746       if (hbckInfo == null) {
1747         regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1748       } else {
1749         hbckInfo.metaEntry = m;
1750       }
1751     }
1752     return true;
1753   }
1754 
1755   private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1756     return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1757       @Override
1758       public void abort(String why, Throwable e) {
1759         LOG.error(why, e);
1760         System.exit(1);
1761       }
1762 
1763       @Override
1764       public boolean isAborted() {
1765         return false;
1766       }
1767 
1768     });
1769   }
1770 
1771   private ServerName getMetaRegionServerName(int replicaId)
1772   throws IOException, KeeperException {
1773     ZooKeeperWatcher zkw = createZooKeeperWatcher();
1774     ServerName sn = null;
1775     try {
1776       sn = new MetaTableLocator().getMetaRegionLocation(zkw, replicaId);
1777     } finally {
1778       zkw.close();
1779     }
1780     return sn;
1781   }
1782 
1783   /**
1784    * Contacts each regionserver and fetches metadata about regions.
1785    * @param regionServerList - the list of region servers to connect to
1786    * @throws IOException if a remote or network exception occurs
1787    */
1788   void processRegionServers(Collection<ServerName> regionServerList)
1789     throws IOException, InterruptedException {
1790 
1791     List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1792     List<Future<Void>> workFutures;
1793 
1794     // loop to contact each region server in parallel
1795     for (ServerName rsinfo: regionServerList) {
1796       workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1797     }
1798 
1799     workFutures = executor.invokeAll(workItems);
1800 
1801     for(int i=0; i<workFutures.size(); i++) {
1802       WorkItemRegion item = workItems.get(i);
1803       Future<Void> f = workFutures.get(i);
1804       try {
1805         f.get();
1806       } catch(ExecutionException e) {
1807         LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1808             e.getCause());
1809       }
1810     }
1811   }
1812 
1813   /**
1814    * Check consistency of all regions that have been found in previous phases.
1815    */
1816   private void checkAndFixConsistency()
1817   throws IOException, KeeperException, InterruptedException {
1818     // Divide the checks in two phases. One for default/primary replicas and another
1819     // for the non-primary ones. Keeps code cleaner this way.
1820 
1821     List<CheckRegionConsistencyWorkItem> workItems =
1822         new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1823     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1824       if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1825         workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1826       }
1827     }
1828     checkRegionConsistencyConcurrently(workItems);
1829 
1830     boolean prevHdfsCheck = shouldCheckHdfs();
1831     setCheckHdfs(false); //replicas don't have any hdfs data
1832     // Run a pass over the replicas and fix any assignment issues that exist on the currently
1833     // deployed/undeployed replicas.
1834     List<CheckRegionConsistencyWorkItem> replicaWorkItems =
1835         new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1836     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1837       if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1838         replicaWorkItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1839       }
1840     }
1841     checkRegionConsistencyConcurrently(replicaWorkItems);
1842     setCheckHdfs(prevHdfsCheck);
1843 
1844     // If some regions is skipped during checkRegionConsistencyConcurrently() phase, we might
1845     // not get accurate state of the hbase if continuing. The config here allows users to tune
1846     // the tolerance of number of skipped region.
1847     // TODO: evaluate the consequence to continue the hbck operation without config.
1848     int terminateThreshold =  getConf().getInt("hbase.hbck.skipped.regions.limit", 0);
1849     int numOfSkippedRegions = skippedRegions.size();
1850     if (numOfSkippedRegions > 0 && numOfSkippedRegions > terminateThreshold) {
1851       throw new IOException(numOfSkippedRegions
1852         + " region(s) could not be checked or repaired.  See logs for detail.");
1853     }
1854 
1855     if (shouldCheckHdfs()) {
1856       checkAndFixTableStates();
1857     }
1858   }
1859 
1860   /**
1861    * Check consistency of all regions using mulitple threads concurrently.
1862    */
1863   private void checkRegionConsistencyConcurrently(
1864     final List<CheckRegionConsistencyWorkItem> workItems)
1865     throws IOException, KeeperException, InterruptedException {
1866     if (workItems.isEmpty()) {
1867       return;  // nothing to check
1868     }
1869 
1870     List<Future<Void>> workFutures = executor.invokeAll(workItems);
1871     for(Future<Void> f: workFutures) {
1872       try {
1873         f.get();
1874       } catch(ExecutionException e1) {
1875         LOG.warn("Could not check region consistency " , e1.getCause());
1876         if (e1.getCause() instanceof IOException) {
1877           throw (IOException)e1.getCause();
1878         } else if (e1.getCause() instanceof KeeperException) {
1879           throw (KeeperException)e1.getCause();
1880         } else if (e1.getCause() instanceof InterruptedException) {
1881           throw (InterruptedException)e1.getCause();
1882         } else {
1883           throw new IOException(e1.getCause());
1884         }
1885       }
1886     }
1887   }
1888 
1889   class CheckRegionConsistencyWorkItem implements Callable<Void> {
1890     private final String key;
1891     private final HbckInfo hbi;
1892 
1893     CheckRegionConsistencyWorkItem(String key, HbckInfo hbi) {
1894       this.key = key;
1895       this.hbi = hbi;
1896     }
1897 
1898     @Override
1899     public synchronized Void call() throws Exception {
1900       try {
1901         checkRegionConsistency(key, hbi);
1902       } catch (Exception e) {
1903         // If the region is non-META region, skip this region and send warning/error message; if
1904         // the region is META region, we should not continue.
1905         LOG.warn("Unable to complete check or repair the region '" + hbi.getRegionNameAsString()
1906           + "'.", e);
1907         if (hbi.getHdfsHRI().isMetaRegion()) {
1908           throw e;
1909         }
1910         LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'");
1911         addSkippedRegion(hbi);
1912       }
1913       return null;
1914     }
1915   }
1916 
1917   private void addSkippedRegion(final HbckInfo hbi) {
1918     Set<String> skippedRegionNames = skippedRegions.get(hbi.getTableName());
1919     if (skippedRegionNames == null) {
1920       skippedRegionNames = new HashSet<String>();
1921     }
1922     skippedRegionNames.add(hbi.getRegionNameAsString());
1923     skippedRegions.put(hbi.getTableName(), skippedRegionNames);
1924   }
1925 
1926   /**
1927    * Check and fix table states, assumes full info available:
1928    * - tableInfos
1929    * - empty tables loaded
1930    */
1931   private void checkAndFixTableStates() throws IOException {
1932     // first check dangling states
1933     for (Entry<TableName, TableState> entry : tableStates.entrySet()) {
1934       TableName tableName = entry.getKey();
1935       TableState tableState = entry.getValue();
1936       TableInfo tableInfo = tablesInfo.get(tableName);
1937       if (isTableIncluded(tableName)
1938           && !tableName.isSystemTable()
1939           && tableInfo == null) {
1940         if (fixMeta) {
1941           MetaTableAccessor.deleteTableState(connection, tableName);
1942           TableState state = MetaTableAccessor.getTableState(connection, tableName);
1943           if (state != null) {
1944             errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,
1945                 tableName + " unable to delete dangling table state " + tableState);
1946           }
1947         } else {
1948           errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,
1949               tableName + " has dangling table state " + tableState);
1950         }
1951       }
1952     }
1953     // check that all tables have states
1954     for (TableName tableName : tablesInfo.keySet()) {
1955       if (isTableIncluded(tableName) && !tableStates.containsKey(tableName)) {
1956         if (fixMeta) {
1957           MetaTableAccessor.updateTableState(connection, tableName, TableState.State.ENABLED);
1958           TableState newState = MetaTableAccessor.getTableState(connection, tableName);
1959           if (newState == null) {
1960             errors.reportError(ERROR_CODE.NO_TABLE_STATE,
1961                 "Unable to change state for table " + tableName + " in meta ");
1962           }
1963         } else {
1964           errors.reportError(ERROR_CODE.NO_TABLE_STATE,
1965               tableName + " has no state in meta ");
1966         }
1967       }
1968     }
1969   }
1970 
1971   private void preCheckPermission() throws IOException, AccessDeniedException {
1972     if (shouldIgnorePreCheckPermission()) {
1973       return;
1974     }
1975 
1976     Path hbaseDir = FSUtils.getRootDir(getConf());
1977     FileSystem fs = hbaseDir.getFileSystem(getConf());
1978     UserProvider userProvider = UserProvider.instantiate(getConf());
1979     UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1980     FileStatus[] files = fs.listStatus(hbaseDir);
1981     for (FileStatus file : files) {
1982       try {
1983         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1984       } catch (AccessDeniedException ace) {
1985         LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1986         errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1987           + " does not have write perms to " + file.getPath()
1988           + ". Please rerun hbck as hdfs user " + file.getOwner());
1989         throw ace;
1990       }
1991     }
1992   }
1993 
1994   /**
1995    * Deletes region from meta table
1996    */
1997   private void deleteMetaRegion(HbckInfo hi) throws IOException {
1998     deleteMetaRegion(hi.metaEntry.getRegionName());
1999   }
2000 
2001   /**
2002    * Deletes region from meta table
2003    */
2004   private void deleteMetaRegion(byte[] metaKey) throws IOException {
2005     Delete d = new Delete(metaKey);
2006     meta.delete(d);
2007     LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
2008   }
2009 
2010   /**
2011    * Reset the split parent region info in meta table
2012    */
2013   private void resetSplitParent(HbckInfo hi) throws IOException {
2014     RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
2015     Delete d = new Delete(hi.metaEntry.getRegionName());
2016     d.addColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
2017     d.addColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
2018     mutations.add(d);
2019 
2020     HRegionInfo hri = new HRegionInfo(hi.metaEntry);
2021     hri.setOffline(false);
2022     hri.setSplit(false);
2023     Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
2024     mutations.add(p);
2025 
2026     meta.mutateRow(mutations);
2027     LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
2028   }
2029 
2030   /**
2031    * This backwards-compatibility wrapper for permanently offlining a region
2032    * that should not be alive.  If the region server does not support the
2033    * "offline" method, it will use the closest unassign method instead.  This
2034    * will basically work until one attempts to disable or delete the affected
2035    * table.  The problem has to do with in-memory only master state, so
2036    * restarting the HMaster or failing over to another should fix this.
2037    */
2038   private void offline(byte[] regionName) throws IOException {
2039     String regionString = Bytes.toStringBinary(regionName);
2040     if (!rsSupportsOffline) {
2041       LOG.warn("Using unassign region " + regionString
2042           + " instead of using offline method, you should"
2043           + " restart HMaster after these repairs");
2044       admin.unassign(regionName, true);
2045       return;
2046     }
2047 
2048     // first time we assume the rs's supports #offline.
2049     try {
2050       LOG.info("Offlining region " + regionString);
2051       admin.offline(regionName);
2052     } catch (IOException ioe) {
2053       String notFoundMsg = "java.lang.NoSuchMethodException: " +
2054         "org.apache.hadoop.hbase.master.HMaster.offline([B)";
2055       if (ioe.getMessage().contains(notFoundMsg)) {
2056         LOG.warn("Using unassign region " + regionString
2057             + " instead of using offline method, you should"
2058             + " restart HMaster after these repairs");
2059         rsSupportsOffline = false; // in the future just use unassign
2060         admin.unassign(regionName, true);
2061         return;
2062       }
2063       throw ioe;
2064     }
2065   }
2066 
2067   private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
2068     undeployRegionsForHbi(hi);
2069     // undeploy replicas of the region (but only if the method is invoked for the primary)
2070     if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2071       return;
2072     }
2073     int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2074     for (int i = 1; i < numReplicas; i++) {
2075       if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
2076       HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
2077           hi.getPrimaryHRIForDeployedReplica(), i);
2078       HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2079       if (h != null) {
2080         undeployRegionsForHbi(h);
2081         //set skip checks; we undeployed it, and we don't want to evaluate this anymore
2082         //in consistency checks
2083         h.setSkipChecks(true);
2084       }
2085     }
2086   }
2087 
2088   private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
2089     for (OnlineEntry rse : hi.deployedEntries) {
2090       LOG.debug("Undeploy region "  + rse.hri + " from " + rse.hsa);
2091       try {
2092         HBaseFsckRepair.closeRegionSilentlyAndWait(connection, rse.hsa, rse.hri);
2093         offline(rse.hri.getRegionName());
2094       } catch (IOException ioe) {
2095         LOG.warn("Got exception when attempting to offline region "
2096             + Bytes.toString(rse.hri.getRegionName()), ioe);
2097       }
2098     }
2099   }
2100 
2101   /**
2102    * Attempts to undeploy a region from a region server based in information in
2103    * META.  Any operations that modify the file system should make sure that
2104    * its corresponding region is not deployed to prevent data races.
2105    *
2106    * A separate call is required to update the master in-memory region state
2107    * kept in the AssignementManager.  Because disable uses this state instead of
2108    * that found in META, we can't seem to cleanly disable/delete tables that
2109    * have been hbck fixed.  When used on a version of HBase that does not have
2110    * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master
2111    * restart or failover may be required.
2112    */
2113   private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
2114     if (hi.metaEntry == null && hi.hdfsEntry == null) {
2115       undeployRegions(hi);
2116       return;
2117     }
2118 
2119     // get assignment info and hregioninfo from meta.
2120     Get get = new Get(hi.getRegionName());
2121     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2122     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
2123     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
2124     // also get the locations of the replicas to close if the primary region is being closed
2125     if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2126       int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2127       for (int i = 0; i < numReplicas; i++) {
2128         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
2129         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
2130       }
2131     }
2132     Result r = meta.get(get);
2133     RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
2134     if (rl == null) {
2135       LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
2136           " since meta does not have handle to reach it");
2137       return;
2138     }
2139     for (HRegionLocation h : rl.getRegionLocations()) {
2140       ServerName serverName = h.getServerName();
2141       if (serverName == null) {
2142         errors.reportError("Unable to close region "
2143             + hi.getRegionNameAsString() +  " because meta does not "
2144             + "have handle to reach it.");
2145         continue;
2146       }
2147       HRegionInfo hri = h.getRegionInfo();
2148       if (hri == null) {
2149         LOG.warn("Unable to close region " + hi.getRegionNameAsString()
2150             + " because hbase:meta had invalid or missing "
2151             + HConstants.CATALOG_FAMILY_STR + ":"
2152             + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
2153             + " qualifier value.");
2154         continue;
2155       }
2156       // close the region -- close files and remove assignment
2157       HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
2158     }
2159   }
2160 
2161   private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
2162     KeeperException, InterruptedException {
2163     // If we are trying to fix the errors
2164     if (shouldFixAssignments()) {
2165       errors.print(msg);
2166       undeployRegions(hbi);
2167       setShouldRerun();
2168       HRegionInfo hri = hbi.getHdfsHRI();
2169       if (hri == null) {
2170         hri = hbi.metaEntry;
2171       }
2172       HBaseFsckRepair.fixUnassigned(admin, hri);
2173       HBaseFsckRepair.waitUntilAssigned(admin, hri);
2174 
2175       // also assign replicas if needed (do it only when this call operates on a primary replica)
2176       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
2177       int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
2178       for (int i = 1; i < replicationCount; i++) {
2179         hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
2180         HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2181         if (h != null) {
2182           undeployRegions(h);
2183           //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore
2184           //in consistency checks
2185           h.setSkipChecks(true);
2186         }
2187         HBaseFsckRepair.fixUnassigned(admin, hri);
2188         HBaseFsckRepair.waitUntilAssigned(admin, hri);
2189       }
2190 
2191     }
2192   }
2193 
2194   /**
2195    * Check a single region for consistency and correct deployment.
2196    */
2197   private void checkRegionConsistency(final String key, final HbckInfo hbi)
2198   throws IOException, KeeperException, InterruptedException {
2199 
2200     if (hbi.isSkipChecks()) return;
2201     String descriptiveName = hbi.toString();
2202     boolean inMeta = hbi.metaEntry != null;
2203     // In case not checking HDFS, assume the region is on HDFS
2204     boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
2205     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
2206     boolean isDeployed = !hbi.deployedOn.isEmpty();
2207     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
2208     boolean deploymentMatchesMeta =
2209       hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
2210       hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
2211     boolean splitParent =
2212         inMeta && hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
2213     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry.getTable());
2214     boolean recentlyModified = inHdfs &&
2215       hbi.getModTime() + timelag > EnvironmentEdgeManager.currentTime();
2216 
2217     // ========== First the healthy cases =============
2218     if (hbi.containsOnlyHdfsEdits()) {
2219       return;
2220     }
2221     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
2222       return;
2223     } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
2224       LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
2225         "tabled that is not deployed");
2226       return;
2227     } else if (recentlyModified) {
2228       LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
2229       return;
2230     }
2231     // ========== Cases where the region is not in hbase:meta =============
2232     else if (!inMeta && !inHdfs && !isDeployed) {
2233       // We shouldn't have record of this region at all then!
2234       assert false : "Entry for region with no data";
2235     } else if (!inMeta && !inHdfs && isDeployed) {
2236       errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
2237           + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
2238           "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2239       if (shouldFixAssignments()) {
2240         undeployRegions(hbi);
2241       }
2242 
2243     } else if (!inMeta && inHdfs && !isDeployed) {
2244       if (hbi.isMerged()) {
2245         // This region has already been merged, the remaining hdfs file will be
2246         // cleaned by CatalogJanitor later
2247         hbi.setSkipChecks(true);
2248         LOG.info("Region " + descriptiveName
2249             + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
2250         return;
2251       }
2252       errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
2253           + descriptiveName + " on HDFS, but not listed in hbase:meta " +
2254           "or deployed on any region server");
2255       // restore region consistency of an adopted orphan
2256       if (shouldFixMeta()) {
2257         if (!hbi.isHdfsRegioninfoPresent()) {
2258           LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
2259               +  " in table integrity repair phase if -fixHdfsOrphans was" +
2260               " used.");
2261           return;
2262         }
2263 
2264         HRegionInfo hri = hbi.getHdfsHRI();
2265         TableInfo tableInfo = tablesInfo.get(hri.getTable());
2266 
2267         for (HRegionInfo region : tableInfo.getRegionsFromMeta()) {
2268           if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0
2269               && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(),
2270                 hri.getEndKey()) >= 0)
2271               && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) {
2272             if(region.isSplit() || region.isOffline()) continue;
2273             Path regionDir = hbi.getHdfsRegionDir();
2274             FileSystem fs = regionDir.getFileSystem(getConf());
2275             List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir);
2276             for (Path familyDir : familyDirs) {
2277               List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);
2278               for (Path referenceFilePath : referenceFilePaths) {
2279                 Path parentRegionDir =
2280                     StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();
2281                 if (parentRegionDir.toString().endsWith(region.getEncodedName())) {
2282                   LOG.warn(hri + " start and stop keys are in the range of " + region
2283                       + ". The region might not be cleaned up from hdfs when region " + region
2284                       + " split failed. Hence deleting from hdfs.");
2285                   HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
2286                     regionDir.getParent(), hri);
2287                   return;
2288                 }
2289               }
2290             }
2291           }
2292         }
2293         LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
2294         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2295         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2296             admin.getClusterStatus().getServers(), numReplicas);
2297 
2298         tryAssignmentRepair(hbi, "Trying to reassign region...");
2299       }
2300 
2301     } else if (!inMeta && inHdfs && isDeployed) {
2302       errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
2303           + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2304       debugLsr(hbi.getHdfsRegionDir());
2305       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2306         // for replicas, this means that we should undeploy the region (we would have
2307         // gone over the primaries and fixed meta holes in first phase under
2308         // checkAndFixConsistency; we shouldn't get the condition !inMeta at
2309         // this stage unless unwanted replica)
2310         if (shouldFixAssignments()) {
2311           undeployRegionsForHbi(hbi);
2312         }
2313       }
2314       if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2315         if (!hbi.isHdfsRegioninfoPresent()) {
2316           LOG.error("This should have been repaired in table integrity repair phase");
2317           return;
2318         }
2319 
2320         LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
2321         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2322         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2323             admin.getClusterStatus().getServers(), numReplicas);
2324         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2325       }
2326 
2327     // ========== Cases where the region is in hbase:meta =============
2328     } else if (inMeta && inHdfs && !isDeployed && splitParent) {
2329       // check whether this is an actual error, or just transient state where parent
2330       // is not cleaned
2331       if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
2332         // check that split daughters are there
2333         HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
2334         HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
2335         if (infoA != null && infoB != null) {
2336           // we already processed or will process daughters. Move on, nothing to see here.
2337           hbi.setSkipChecks(true);
2338           return;
2339         }
2340       }
2341       errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2342           + descriptiveName + " is a split parent in META, in HDFS, "
2343           + "and not deployed on any region server. This could be transient.");
2344       if (shouldFixSplitParents()) {
2345         setShouldRerun();
2346         resetSplitParent(hbi);
2347       }
2348     } else if (inMeta && !inHdfs && !isDeployed) {
2349       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2350           + descriptiveName + " found in META, but not in HDFS "
2351           + "or deployed on any region server.");
2352       if (shouldFixMeta()) {
2353         deleteMetaRegion(hbi);
2354       }
2355     } else if (inMeta && !inHdfs && isDeployed) {
2356       errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2357           + " found in META, but not in HDFS, " +
2358           "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2359       // We treat HDFS as ground truth.  Any information in meta is transient
2360       // and equivalent data can be regenerated.  So, lets unassign and remove
2361       // these problems from META.
2362       if (shouldFixAssignments()) {
2363         errors.print("Trying to fix unassigned region...");
2364         undeployRegions(hbi);
2365       }
2366       if (shouldFixMeta()) {
2367         // wait for it to complete
2368         deleteMetaRegion(hbi);
2369       }
2370     } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2371       errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2372           + " not deployed on any region server.");
2373       tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2374     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2375       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2376           "Region " + descriptiveName + " should not be deployed according " +
2377           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2378       if (shouldFixAssignments()) {
2379         errors.print("Trying to close the region " + descriptiveName);
2380         setShouldRerun();
2381         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2382       }
2383     } else if (inMeta && inHdfs && isMultiplyDeployed) {
2384       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2385           + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
2386           + " but is multiply assigned to region servers " +
2387           Joiner.on(", ").join(hbi.deployedOn));
2388       // If we are trying to fix the errors
2389       if (shouldFixAssignments()) {
2390         errors.print("Trying to fix assignment error...");
2391         setShouldRerun();
2392         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2393       }
2394     } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2395       errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2396           + descriptiveName + " listed in hbase:meta on region server " +
2397           hbi.metaEntry.regionServer + " but found on region server " +
2398           hbi.deployedOn.get(0));
2399       // If we are trying to fix the errors
2400       if (shouldFixAssignments()) {
2401         errors.print("Trying to fix assignment error...");
2402         setShouldRerun();
2403         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2404         HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2405       }
2406     } else {
2407       errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2408           " is in an unforeseen state:" +
2409           " inMeta=" + inMeta +
2410           " inHdfs=" + inHdfs +
2411           " isDeployed=" + isDeployed +
2412           " isMultiplyDeployed=" + isMultiplyDeployed +
2413           " deploymentMatchesMeta=" + deploymentMatchesMeta +
2414           " shouldBeDeployed=" + shouldBeDeployed);
2415     }
2416   }
2417 
2418   /**
2419    * Checks tables integrity. Goes over all regions and scans the tables.
2420    * Collects all the pieces for each table and checks if there are missing,
2421    * repeated or overlapping ones.
2422    * @throws IOException
2423    */
2424   SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2425     tablesInfo = new TreeMap<TableName,TableInfo> ();
2426     LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2427     for (HbckInfo hbi : regionInfoMap.values()) {
2428       // Check only valid, working regions
2429       if (hbi.metaEntry == null) {
2430         // this assumes that consistency check has run loadMetaEntry
2431         Path p = hbi.getHdfsRegionDir();
2432         if (p == null) {
2433           errors.report("No regioninfo in Meta or HDFS. " + hbi);
2434         }
2435 
2436         // TODO test.
2437         continue;
2438       }
2439       if (hbi.metaEntry.regionServer == null) {
2440         errors.detail("Skipping region because no region server: " + hbi);
2441         continue;
2442       }
2443       if (hbi.metaEntry.isOffline()) {
2444         errors.detail("Skipping region because it is offline: " + hbi);
2445         continue;
2446       }
2447       if (hbi.containsOnlyHdfsEdits()) {
2448         errors.detail("Skipping region because it only contains edits" + hbi);
2449         continue;
2450       }
2451 
2452       // Missing regionDir or over-deployment is checked elsewhere. Include
2453       // these cases in modTInfo, so we can evaluate those regions as part of
2454       // the region chain in META
2455       //if (hbi.foundRegionDir == null) continue;
2456       //if (hbi.deployedOn.size() != 1) continue;
2457       if (hbi.deployedOn.size() == 0) continue;
2458 
2459       // We should be safe here
2460       TableName tableName = hbi.metaEntry.getTable();
2461       TableInfo modTInfo = tablesInfo.get(tableName);
2462       if (modTInfo == null) {
2463         modTInfo = new TableInfo(tableName);
2464       }
2465       for (ServerName server : hbi.deployedOn) {
2466         modTInfo.addServer(server);
2467       }
2468 
2469       if (!hbi.isSkipChecks()) {
2470         modTInfo.addRegionInfo(hbi);
2471       }
2472 
2473       tablesInfo.put(tableName, modTInfo);
2474     }
2475 
2476     loadTableInfosForTablesWithNoRegion();
2477 
2478     logParallelMerge();
2479     for (TableInfo tInfo : tablesInfo.values()) {
2480       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2481       if (!tInfo.checkRegionChain(handler)) {
2482         errors.report("Found inconsistency in table " + tInfo.getName());
2483       }
2484     }
2485     return tablesInfo;
2486   }
2487 
2488   /** Loads table info's for tables that may not have been included, since there are no
2489    * regions reported for the table, but table dir is there in hdfs
2490    */
2491   private void loadTableInfosForTablesWithNoRegion() throws IOException {
2492     Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2493     for (HTableDescriptor htd : allTables.values()) {
2494       if (checkMetaOnly && !htd.isMetaTable()) {
2495         continue;
2496       }
2497 
2498       TableName tableName = htd.getTableName();
2499       if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2500         TableInfo tableInfo = new TableInfo(tableName);
2501         tableInfo.htds.add(htd);
2502         tablesInfo.put(htd.getTableName(), tableInfo);
2503       }
2504     }
2505   }
2506 
2507   /**
2508    * Merge hdfs data by moving from contained HbckInfo into targetRegionDir.
2509    * @return number of file move fixes done to merge regions.
2510    */
2511   public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2512     int fileMoves = 0;
2513     String thread = Thread.currentThread().getName();
2514     LOG.debug("[" + thread + "] Contained region dir after close and pause");
2515     debugLsr(contained.getHdfsRegionDir());
2516 
2517     // rename the contained into the container.
2518     FileSystem fs = targetRegionDir.getFileSystem(getConf());
2519     FileStatus[] dirs = null;
2520     try {
2521       dirs = fs.listStatus(contained.getHdfsRegionDir());
2522     } catch (FileNotFoundException fnfe) {
2523       // region we are attempting to merge in is not present!  Since this is a merge, there is
2524       // no harm skipping this region if it does not exist.
2525       if (!fs.exists(contained.getHdfsRegionDir())) {
2526         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2527             + " is missing. Assuming already sidelined or moved.");
2528       } else {
2529         sidelineRegionDir(fs, contained);
2530       }
2531       return fileMoves;
2532     }
2533 
2534     if (dirs == null) {
2535       if (!fs.exists(contained.getHdfsRegionDir())) {
2536         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2537             + " already sidelined.");
2538       } else {
2539         sidelineRegionDir(fs, contained);
2540       }
2541       return fileMoves;
2542     }
2543 
2544     for (FileStatus cf : dirs) {
2545       Path src = cf.getPath();
2546       Path dst =  new Path(targetRegionDir, src.getName());
2547 
2548       if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2549         // do not copy the old .regioninfo file.
2550         continue;
2551       }
2552 
2553       if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2554         // do not copy the .oldlogs files
2555         continue;
2556       }
2557 
2558       LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2559       // FileSystem.rename is inconsistent with directories -- if the
2560       // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2561       // it moves the src into the dst dir resulting in (foo/a/b).  If
2562       // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2563       for (FileStatus hfile : fs.listStatus(src)) {
2564         boolean success = fs.rename(hfile.getPath(), dst);
2565         if (success) {
2566           fileMoves++;
2567         }
2568       }
2569       LOG.debug("[" + thread + "] Sideline directory contents:");
2570       debugLsr(targetRegionDir);
2571     }
2572 
2573     // if all success.
2574     sidelineRegionDir(fs, contained);
2575     LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2576         getSidelineDir());
2577     debugLsr(contained.getHdfsRegionDir());
2578 
2579     return fileMoves;
2580   }
2581 
2582 
2583   static class WorkItemOverlapMerge implements Callable<Void> {
2584     private TableIntegrityErrorHandler handler;
2585     Collection<HbckInfo> overlapgroup;
2586 
2587     WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2588       this.handler = handler;
2589       this.overlapgroup = overlapgroup;
2590     }
2591 
2592     @Override
2593     public Void call() throws Exception {
2594       handler.handleOverlapGroup(overlapgroup);
2595       return null;
2596     }
2597   };
2598 
2599 
2600   /**
2601    * Maintain information about a particular table.
2602    */
2603   public class TableInfo {
2604     TableName tableName;
2605     TreeSet <ServerName> deployedOn;
2606 
2607     // backwards regions
2608     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2609 
2610     // sidelined big overlapped regions
2611     final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2612 
2613     // region split calculator
2614     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2615 
2616     // Histogram of different HTableDescriptors found.  Ideally there is only one!
2617     final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2618 
2619     // key = start split, values = set of splits in problem group
2620     final Multimap<byte[], HbckInfo> overlapGroups =
2621       TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2622 
2623     // list of regions derived from meta entries.
2624     private ImmutableList<HRegionInfo> regionsFromMeta = null;
2625 
2626     TableInfo(TableName name) {
2627       this.tableName = name;
2628       deployedOn = new TreeSet <ServerName>();
2629     }
2630 
2631     /**
2632      * @return descriptor common to all regions.  null if are none or multiple!
2633      */
2634     private HTableDescriptor getHTD() {
2635       if (htds.size() == 1) {
2636         return (HTableDescriptor)htds.toArray()[0];
2637       } else {
2638         LOG.error("None/Multiple table descriptors found for table '"
2639           + tableName + "' regions: " + htds);
2640       }
2641       return null;
2642     }
2643 
2644     public void addRegionInfo(HbckInfo hir) {
2645       if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2646         // end key is absolute end key, just add it.
2647         // ignore replicas other than primary for these checks
2648         if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2649         return;
2650       }
2651 
2652       // if not the absolute end key, check for cycle
2653       if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2654         errors.reportError(
2655             ERROR_CODE.REGION_CYCLE,
2656             String.format("The endkey for this region comes before the "
2657                 + "startkey, startkey=%s, endkey=%s",
2658                 Bytes.toStringBinary(hir.getStartKey()),
2659                 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2660         backwards.add(hir);
2661         return;
2662       }
2663 
2664       // main case, add to split calculator
2665       // ignore replicas other than primary for these checks
2666       if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2667     }
2668 
2669     public void addServer(ServerName server) {
2670       this.deployedOn.add(server);
2671     }
2672 
2673     public TableName getName() {
2674       return tableName;
2675     }
2676 
2677     public int getNumRegions() {
2678       return sc.getStarts().size() + backwards.size();
2679     }
2680 
2681     public synchronized ImmutableList<HRegionInfo> getRegionsFromMeta() {
2682       // lazy loaded, synchronized to ensure a single load
2683       if (regionsFromMeta == null) {
2684         List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
2685         for (HbckInfo h : HBaseFsck.this.regionInfoMap.values()) {
2686           if (tableName.equals(h.getTableName())) {
2687             if (h.metaEntry != null) {
2688               regions.add((HRegionInfo) h.metaEntry);
2689             }
2690           }
2691         }
2692         regionsFromMeta = Ordering.natural().immutableSortedCopy(regions);
2693       }
2694 
2695       return regionsFromMeta;
2696     }
2697 
2698     private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2699       ErrorReporter errors;
2700 
2701       IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2702         this.errors = errors;
2703         setTableInfo(ti);
2704       }
2705 
2706       @Override
2707       public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2708         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2709             "First region should start with an empty key.  You need to "
2710             + " create a new region and regioninfo in HDFS to plug the hole.",
2711             getTableInfo(), hi);
2712       }
2713 
2714       @Override
2715       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2716         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2717             "Last region should end with an empty key. You need to "
2718                 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2719       }
2720 
2721       @Override
2722       public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2723         errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2724             "Region has the same start and end key.", getTableInfo(), hi);
2725       }
2726 
2727       @Override
2728       public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2729         byte[] key = r1.getStartKey();
2730         // dup start key
2731         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2732             "Multiple regions have the same startkey: "
2733             + Bytes.toStringBinary(key), getTableInfo(), r1);
2734         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2735             "Multiple regions have the same startkey: "
2736             + Bytes.toStringBinary(key), getTableInfo(), r2);
2737       }
2738 
2739       @Override
2740       public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2741         errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2742             "There is an overlap in the region chain.",
2743             getTableInfo(), hi1, hi2);
2744       }
2745 
2746       @Override
2747       public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2748         errors.reportError(
2749             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2750             "There is a hole in the region chain between "
2751                 + Bytes.toStringBinary(holeStart) + " and "
2752                 + Bytes.toStringBinary(holeStop)
2753                 + ".  You need to create a new .regioninfo and region "
2754                 + "dir in hdfs to plug the hole.");
2755       }
2756     };
2757 
2758     /**
2759      * This handler fixes integrity errors from hdfs information.  There are
2760      * basically three classes of integrity problems 1) holes, 2) overlaps, and
2761      * 3) invalid regions.
2762      *
2763      * This class overrides methods that fix holes and the overlap group case.
2764      * Individual cases of particular overlaps are handled by the general
2765      * overlap group merge repair case.
2766      *
2767      * If hbase is online, this forces regions offline before doing merge
2768      * operations.
2769      */
2770     private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2771       Configuration conf;
2772 
2773       boolean fixOverlaps = true;
2774 
2775       HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2776           boolean fixHoles, boolean fixOverlaps) {
2777         super(ti, errors);
2778         this.conf = conf;
2779         this.fixOverlaps = fixOverlaps;
2780         // TODO properly use fixHoles
2781       }
2782 
2783       /**
2784        * This is a special case hole -- when the first region of a table is
2785        * missing from META, HBase doesn't acknowledge the existance of the
2786        * table.
2787        */
2788       @Override
2789       public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2790         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2791             "First region should start with an empty key.  Creating a new " +
2792             "region and regioninfo in HDFS to plug the hole.",
2793             getTableInfo(), next);
2794         HTableDescriptor htd = getTableInfo().getHTD();
2795         // from special EMPTY_START_ROW to next region's startKey
2796         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2797             HConstants.EMPTY_START_ROW, next.getStartKey());
2798 
2799         // TODO test
2800         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2801         LOG.info("Table region start key was not empty.  Created new empty region: "
2802             + newRegion + " " +region);
2803         fixes++;
2804       }
2805 
2806       @Override
2807       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2808         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2809             "Last region should end with an empty key.  Creating a new "
2810                 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2811         HTableDescriptor htd = getTableInfo().getHTD();
2812         // from curEndKey to EMPTY_START_ROW
2813         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2814             HConstants.EMPTY_START_ROW);
2815 
2816         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2817         LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
2818             + " " + region);
2819         fixes++;
2820       }
2821 
2822       /**
2823        * There is a hole in the hdfs regions that violates the table integrity
2824        * rules.  Create a new empty region that patches the hole.
2825        */
2826       @Override
2827       public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2828         errors.reportError(
2829             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2830             "There is a hole in the region chain between "
2831                 + Bytes.toStringBinary(holeStartKey) + " and "
2832                 + Bytes.toStringBinary(holeStopKey)
2833                 + ".  Creating a new regioninfo and region "
2834                 + "dir in hdfs to plug the hole.");
2835         HTableDescriptor htd = getTableInfo().getHTD();
2836         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2837         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2838         LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2839         fixes++;
2840       }
2841 
2842       /**
2843        * This takes set of overlapping regions and merges them into a single
2844        * region.  This covers cases like degenerate regions, shared start key,
2845        * general overlaps, duplicate ranges, and partial overlapping regions.
2846        *
2847        * Cases:
2848        * - Clean regions that overlap
2849        * - Only .oldlogs regions (can't find start/stop range, or figure out)
2850        *
2851        * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
2852        */
2853       @Override
2854       public void handleOverlapGroup(Collection<HbckInfo> overlap)
2855           throws IOException {
2856         Preconditions.checkNotNull(overlap);
2857         Preconditions.checkArgument(overlap.size() >0);
2858 
2859         if (!this.fixOverlaps) {
2860           LOG.warn("Not attempting to repair overlaps.");
2861           return;
2862         }
2863 
2864         if (overlap.size() > maxMerge) {
2865           LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2866             "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2867           if (sidelineBigOverlaps) {
2868             // we only sideline big overlapped groups that exceeds the max number of regions to merge
2869             sidelineBigOverlaps(overlap);
2870           }
2871           return;
2872         }
2873 
2874         mergeOverlaps(overlap);
2875       }
2876 
2877       void mergeOverlaps(Collection<HbckInfo> overlap)
2878           throws IOException {
2879         String thread = Thread.currentThread().getName();
2880         LOG.info("== [" + thread + "] Merging regions into one region: "
2881           + Joiner.on(",").join(overlap));
2882         // get the min / max range and close all concerned regions
2883         Pair<byte[], byte[]> range = null;
2884         for (HbckInfo hi : overlap) {
2885           if (range == null) {
2886             range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2887           } else {
2888             if (RegionSplitCalculator.BYTES_COMPARATOR
2889                 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2890               range.setFirst(hi.getStartKey());
2891             }
2892             if (RegionSplitCalculator.BYTES_COMPARATOR
2893                 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2894               range.setSecond(hi.getEndKey());
2895             }
2896           }
2897           // need to close files so delete can happen.
2898           LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
2899           LOG.debug("[" + thread + "] Contained region dir before close");
2900           debugLsr(hi.getHdfsRegionDir());
2901           try {
2902             LOG.info("[" + thread + "] Closing region: " + hi);
2903             closeRegion(hi);
2904           } catch (IOException ioe) {
2905             LOG.warn("[" + thread + "] Was unable to close region " + hi
2906               + ".  Just continuing... ", ioe);
2907           } catch (InterruptedException e) {
2908             LOG.warn("[" + thread + "] Was unable to close region " + hi
2909               + ".  Just continuing... ", e);
2910           }
2911 
2912           try {
2913             LOG.info("[" + thread + "] Offlining region: " + hi);
2914             offline(hi.getRegionName());
2915           } catch (IOException ioe) {
2916             LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2917               + ".  Just continuing... ", ioe);
2918           }
2919         }
2920 
2921         // create new empty container region.
2922         HTableDescriptor htd = getTableInfo().getHTD();
2923         // from start key to end Key
2924         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
2925             range.getSecond());
2926         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2927         LOG.info("[" + thread + "] Created new empty container region: " +
2928             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
2929         debugLsr(region.getRegionFileSystem().getRegionDir());
2930 
2931         // all target regions are closed, should be able to safely cleanup.
2932         boolean didFix= false;
2933         Path target = region.getRegionFileSystem().getRegionDir();
2934         for (HbckInfo contained : overlap) {
2935           LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
2936           int merges = mergeRegionDirs(target, contained);
2937           if (merges > 0) {
2938             didFix = true;
2939           }
2940         }
2941         if (didFix) {
2942           fixes++;
2943         }
2944       }
2945 
2946       /**
2947        * Sideline some regions in a big overlap group so that it
2948        * will have fewer regions, and it is easier to merge them later on.
2949        *
2950        * @param bigOverlap the overlapped group with regions more than maxMerge
2951        * @throws IOException
2952        */
2953       void sidelineBigOverlaps(
2954           Collection<HbckInfo> bigOverlap) throws IOException {
2955         int overlapsToSideline = bigOverlap.size() - maxMerge;
2956         if (overlapsToSideline > maxOverlapsToSideline) {
2957           overlapsToSideline = maxOverlapsToSideline;
2958         }
2959         List<HbckInfo> regionsToSideline =
2960           RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
2961         FileSystem fs = FileSystem.get(conf);
2962         for (HbckInfo regionToSideline: regionsToSideline) {
2963           try {
2964             LOG.info("Closing region: " + regionToSideline);
2965             closeRegion(regionToSideline);
2966           } catch (IOException ioe) {
2967             LOG.warn("Was unable to close region " + regionToSideline
2968               + ".  Just continuing... ", ioe);
2969           } catch (InterruptedException e) {
2970             LOG.warn("Was unable to close region " + regionToSideline
2971               + ".  Just continuing... ", e);
2972           }
2973 
2974           try {
2975             LOG.info("Offlining region: " + regionToSideline);
2976             offline(regionToSideline.getRegionName());
2977           } catch (IOException ioe) {
2978             LOG.warn("Unable to offline region from master: " + regionToSideline
2979               + ".  Just continuing... ", ioe);
2980           }
2981 
2982           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
2983           Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
2984           if (sidelineRegionDir != null) {
2985             sidelinedRegions.put(sidelineRegionDir, regionToSideline);
2986             LOG.info("After sidelined big overlapped region: "
2987               + regionToSideline.getRegionNameAsString()
2988               + " to " + sidelineRegionDir.toString());
2989             fixes++;
2990           }
2991         }
2992       }
2993     }
2994 
2995     /**
2996      * Check the region chain (from META) of this table.  We are looking for
2997      * holes, overlaps, and cycles.
2998      * @return false if there are errors
2999      * @throws IOException
3000      */
3001     public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
3002       // When table is disabled no need to check for the region chain. Some of the regions
3003       // accidently if deployed, this below code might report some issues like missing start
3004       // or end regions or region hole in chain and may try to fix which is unwanted.
3005       if (isTableDisabled(this.tableName)) {
3006         return true;
3007       }
3008       int originalErrorsCount = errors.getErrorList().size();
3009       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
3010       SortedSet<byte[]> splits = sc.getSplits();
3011 
3012       byte[] prevKey = null;
3013       byte[] problemKey = null;
3014 
3015       if (splits.size() == 0) {
3016         // no region for this table
3017         handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
3018       }
3019 
3020       for (byte[] key : splits) {
3021         Collection<HbckInfo> ranges = regions.get(key);
3022         if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
3023           for (HbckInfo rng : ranges) {
3024             handler.handleRegionStartKeyNotEmpty(rng);
3025           }
3026         }
3027 
3028         // check for degenerate ranges
3029         for (HbckInfo rng : ranges) {
3030           // special endkey case converts '' to null
3031           byte[] endKey = rng.getEndKey();
3032           endKey = (endKey.length == 0) ? null : endKey;
3033           if (Bytes.equals(rng.getStartKey(),endKey)) {
3034             handler.handleDegenerateRegion(rng);
3035           }
3036         }
3037 
3038         if (ranges.size() == 1) {
3039           // this split key is ok -- no overlap, not a hole.
3040           if (problemKey != null) {
3041             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3042           }
3043           problemKey = null; // fell through, no more problem.
3044         } else if (ranges.size() > 1) {
3045           // set the new problem key group name, if already have problem key, just
3046           // keep using it.
3047           if (problemKey == null) {
3048             // only for overlap regions.
3049             LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
3050             problemKey = key;
3051           }
3052           overlapGroups.putAll(problemKey, ranges);
3053 
3054           // record errors
3055           ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
3056           //  this dumb and n^2 but this shouldn't happen often
3057           for (HbckInfo r1 : ranges) {
3058             if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3059             subRange.remove(r1);
3060             for (HbckInfo r2 : subRange) {
3061               if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3062               if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
3063                 handler.handleDuplicateStartKeys(r1,r2);
3064               } else {
3065                 // overlap
3066                 handler.handleOverlapInRegionChain(r1, r2);
3067               }
3068             }
3069           }
3070 
3071         } else if (ranges.size() == 0) {
3072           if (problemKey != null) {
3073             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3074           }
3075           problemKey = null;
3076 
3077           byte[] holeStopKey = sc.getSplits().higher(key);
3078           // if higher key is null we reached the top.
3079           if (holeStopKey != null) {
3080             // hole
3081             handler.handleHoleInRegionChain(key, holeStopKey);
3082           }
3083         }
3084         prevKey = key;
3085       }
3086 
3087       // When the last region of a table is proper and having an empty end key, 'prevKey'
3088       // will be null.
3089       if (prevKey != null) {
3090         handler.handleRegionEndKeyNotEmpty(prevKey);
3091       }
3092 
3093       // TODO fold this into the TableIntegrityHandler
3094       if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
3095         boolean ok = handleOverlapsParallel(handler, prevKey);
3096         if (!ok) {
3097           return false;
3098         }
3099       } else {
3100         for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3101           handler.handleOverlapGroup(overlap);
3102         }
3103       }
3104 
3105       if (details) {
3106         // do full region split map dump
3107         errors.print("---- Table '"  +  this.tableName
3108             + "': region split map");
3109         dump(splits, regions);
3110         errors.print("---- Table '"  +  this.tableName
3111             + "': overlap groups");
3112         dumpOverlapProblems(overlapGroups);
3113         errors.print("There are " + overlapGroups.keySet().size()
3114             + " overlap groups with " + overlapGroups.size()
3115             + " overlapping regions");
3116       }
3117       if (!sidelinedRegions.isEmpty()) {
3118         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
3119         errors.print("---- Table '"  +  this.tableName
3120             + "': sidelined big overlapped regions");
3121         dumpSidelinedRegions(sidelinedRegions);
3122       }
3123       return errors.getErrorList().size() == originalErrorsCount;
3124     }
3125 
3126     private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
3127         throws IOException {
3128       // we parallelize overlap handler for the case we have lots of groups to fix.  We can
3129       // safely assume each group is independent.
3130       List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
3131       List<Future<Void>> rets;
3132       for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3133         //
3134         merges.add(new WorkItemOverlapMerge(overlap, handler));
3135       }
3136       try {
3137         rets = executor.invokeAll(merges);
3138       } catch (InterruptedException e) {
3139         LOG.error("Overlap merges were interrupted", e);
3140         return false;
3141       }
3142       for(int i=0; i<merges.size(); i++) {
3143         WorkItemOverlapMerge work = merges.get(i);
3144         Future<Void> f = rets.get(i);
3145         try {
3146           f.get();
3147         } catch(ExecutionException e) {
3148           LOG.warn("Failed to merge overlap group" + work, e.getCause());
3149         } catch (InterruptedException e) {
3150           LOG.error("Waiting for overlap merges was interrupted", e);
3151           return false;
3152         }
3153       }
3154       return true;
3155     }
3156 
3157     /**
3158      * This dumps data in a visually reasonable way for visual debugging
3159      *
3160      * @param splits
3161      * @param regions
3162      */
3163     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
3164       // we display this way because the last end key should be displayed as well.
3165       StringBuilder sb = new StringBuilder();
3166       for (byte[] k : splits) {
3167         sb.setLength(0); // clear out existing buffer, if any.
3168         sb.append(Bytes.toStringBinary(k) + ":\t");
3169         for (HbckInfo r : regions.get(k)) {
3170           sb.append("[ "+ r.toString() + ", "
3171               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
3172         }
3173         errors.print(sb.toString());
3174       }
3175     }
3176   }
3177 
3178   public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
3179     // we display this way because the last end key should be displayed as
3180     // well.
3181     for (byte[] k : regions.keySet()) {
3182       errors.print(Bytes.toStringBinary(k) + ":");
3183       for (HbckInfo r : regions.get(k)) {
3184         errors.print("[ " + r.toString() + ", "
3185             + Bytes.toStringBinary(r.getEndKey()) + "]");
3186       }
3187       errors.print("----");
3188     }
3189   }
3190 
3191   public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
3192     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
3193       TableName tableName = entry.getValue().getTableName();
3194       Path path = entry.getKey();
3195       errors.print("This sidelined region dir should be bulk loaded: "
3196         + path.toString());
3197       errors.print("Bulk load command looks like: "
3198         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
3199         + path.toUri().getPath() + " "+ tableName);
3200     }
3201   }
3202 
3203   public Multimap<byte[], HbckInfo> getOverlapGroups(
3204       TableName table) {
3205     TableInfo ti = tablesInfo.get(table);
3206     return ti.overlapGroups;
3207   }
3208 
3209   /**
3210    * Return a list of user-space table names whose metadata have not been
3211    * modified in the last few milliseconds specified by timelag
3212    * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
3213    * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
3214    * milliseconds specified by timelag, then the table is a candidate to be returned.
3215    * @return tables that have not been modified recently
3216    * @throws IOException if an error is encountered
3217    */
3218   HTableDescriptor[] getTables(AtomicInteger numSkipped) {
3219     List<TableName> tableNames = new ArrayList<TableName>();
3220     long now = EnvironmentEdgeManager.currentTime();
3221 
3222     for (HbckInfo hbi : regionInfoMap.values()) {
3223       MetaEntry info = hbi.metaEntry;
3224 
3225       // if the start key is zero, then we have found the first region of a table.
3226       // pick only those tables that were not modified in the last few milliseconds.
3227       if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
3228         if (info.modTime + timelag < now) {
3229           tableNames.add(info.getTable());
3230         } else {
3231           numSkipped.incrementAndGet(); // one more in-flux table
3232         }
3233       }
3234     }
3235     return getHTableDescriptors(tableNames);
3236   }
3237 
3238   HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
3239     HTableDescriptor[] htd = new HTableDescriptor[0];
3240       LOG.info("getHTableDescriptors == tableNames => " + tableNames);
3241     try (Connection conn = ConnectionFactory.createConnection(getConf());
3242         Admin admin = conn.getAdmin()) {
3243       htd = admin.getTableDescriptorsByTableName(tableNames);
3244     } catch (IOException e) {
3245       LOG.debug("Exception getting table descriptors", e);
3246     }
3247     return htd;
3248   }
3249 
3250   /**
3251    * Gets the entry in regionInfo corresponding to the the given encoded
3252    * region name. If the region has not been seen yet, a new entry is added
3253    * and returned.
3254    */
3255   private synchronized HbckInfo getOrCreateInfo(String name) {
3256     HbckInfo hbi = regionInfoMap.get(name);
3257     if (hbi == null) {
3258       hbi = new HbckInfo(null);
3259       regionInfoMap.put(name, hbi);
3260     }
3261     return hbi;
3262   }
3263 
3264   private void checkAndFixTableLocks() throws IOException {
3265     ZooKeeperWatcher zkw = createZooKeeperWatcher();
3266     TableLockChecker checker = new TableLockChecker(zkw, errors);
3267     checker.checkTableLocks();
3268 
3269     if (this.fixTableLocks) {
3270       checker.fixExpiredTableLocks();
3271     }
3272     zkw.close();
3273   }
3274   
3275   private void checkAndFixReplication() throws IOException {
3276     ZooKeeperWatcher zkw = createZooKeeperWatcher();
3277     try {
3278       ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, connection, errors);
3279       checker.checkUnDeletedQueues();
3280 
3281       if (checker.hasUnDeletedQueues() && this.fixReplication) {
3282         checker.fixUnDeletedQueues();
3283         setShouldRerun();
3284       }
3285     } finally {
3286       zkw.close();
3287     }
3288   }
3289 
3290   /**
3291     * Check values in regionInfo for hbase:meta
3292     * Check if zero or more than one regions with hbase:meta are found.
3293     * If there are inconsistencies (i.e. zero or more than one regions
3294     * pretend to be holding the hbase:meta) try to fix that and report an error.
3295     * @throws IOException from HBaseFsckRepair functions
3296     * @throws KeeperException
3297     * @throws InterruptedException
3298     */
3299   boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
3300     Map<Integer, HbckInfo> metaRegions = new HashMap<Integer, HbckInfo>();
3301     for (HbckInfo value : regionInfoMap.values()) {
3302       if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
3303         metaRegions.put(value.getReplicaId(), value);
3304       }
3305     }
3306     int metaReplication = admin.getTableDescriptor(TableName.META_TABLE_NAME)
3307         .getRegionReplication();
3308     boolean noProblem = true;
3309     // There will be always entries in regionInfoMap corresponding to hbase:meta & its replicas
3310     // Check the deployed servers. It should be exactly one server for each replica.
3311     for (int i = 0; i < metaReplication; i++) {
3312       HbckInfo metaHbckInfo = metaRegions.remove(i);
3313       List<ServerName> servers = new ArrayList<ServerName>();
3314       if (metaHbckInfo != null) {
3315         servers = metaHbckInfo.deployedOn;
3316       }
3317       if (servers.size() != 1) {
3318         noProblem = false;
3319         if (servers.size() == 0) {
3320           assignMetaReplica(i);
3321         } else if (servers.size() > 1) {
3322           errors
3323           .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId " +
3324                        metaHbckInfo.getReplicaId() + " is found on more than one region.");
3325           if (shouldFixAssignments()) {
3326             errors.print("Trying to fix a problem with hbase:meta, replicaId " +
3327                          metaHbckInfo.getReplicaId() +"..");
3328             setShouldRerun();
3329             // try fix it (treat is a dupe assignment)
3330             HBaseFsckRepair.fixMultiAssignment(connection, metaHbckInfo.metaEntry, servers);
3331           }
3332         }
3333       }
3334     }
3335     // unassign whatever is remaining in metaRegions. They are excess replicas.
3336     for (Map.Entry<Integer, HbckInfo> entry : metaRegions.entrySet()) {
3337       noProblem = false;
3338       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
3339           "hbase:meta replicas are deployed in excess. Configured " + metaReplication +
3340           ", deployed " + metaRegions.size());
3341       if (shouldFixAssignments()) {
3342         errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() +
3343             " of hbase:meta..");
3344         setShouldRerun();
3345         unassignMetaReplica(entry.getValue());
3346       }
3347     }
3348     // if noProblem is false, rerun hbck with hopefully fixed META
3349     // if noProblem is true, no errors, so continue normally
3350     return noProblem;
3351   }
3352 
3353   private void unassignMetaReplica(HbckInfo hi) throws IOException, InterruptedException,
3354   KeeperException {
3355     undeployRegions(hi);
3356     ZooKeeperWatcher zkw = createZooKeeperWatcher();
3357     try {
3358       ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(hi.metaEntry.getReplicaId()));
3359     } finally {
3360       zkw.close();
3361     }
3362   }
3363 
3364   private void assignMetaReplica(int replicaId)
3365       throws IOException, KeeperException, InterruptedException {
3366     errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " +
3367         replicaId +" is not found on any region.");
3368     if (shouldFixAssignments()) {
3369       errors.print("Trying to fix a problem with hbase:meta..");
3370       setShouldRerun();
3371       // try to fix it (treat it as unassigned region)
3372       HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
3373           HRegionInfo.FIRST_META_REGIONINFO, replicaId);
3374       HBaseFsckRepair.fixUnassigned(admin, h);
3375       HBaseFsckRepair.waitUntilAssigned(admin, h);
3376     }
3377   }
3378 
3379   /**
3380    * Scan hbase:meta, adding all regions found to the regionInfo map.
3381    * @throws IOException if an error is encountered
3382    */
3383   boolean loadMetaEntries() throws IOException {
3384     MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
3385       int countRecord = 1;
3386 
3387       // comparator to sort KeyValues with latest modtime
3388       final Comparator<Cell> comp = new Comparator<Cell>() {
3389         @Override
3390         public int compare(Cell k1, Cell k2) {
3391           return (int)(k1.getTimestamp() - k2.getTimestamp());
3392         }
3393       };
3394 
3395       @Override
3396       public boolean visit(Result result) throws IOException {
3397         try {
3398 
3399           // record the latest modification of this META record
3400           long ts =  Collections.max(result.listCells(), comp).getTimestamp();
3401           RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
3402           if (rl == null) {
3403             emptyRegionInfoQualifiers.add(result);
3404             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3405               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3406             return true;
3407           }
3408           ServerName sn = null;
3409           if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
3410               rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
3411             emptyRegionInfoQualifiers.add(result);
3412             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3413               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3414             return true;
3415           }
3416           HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
3417           if (!(isTableIncluded(hri.getTable())
3418               || hri.isMetaRegion())) {
3419             return true;
3420           }
3421           PairOfSameType<HRegionInfo> daughters = MetaTableAccessor.getDaughterRegions(result);
3422           for (HRegionLocation h : rl.getRegionLocations()) {
3423             if (h == null || h.getRegionInfo() == null) {
3424               continue;
3425             }
3426             sn = h.getServerName();
3427             hri = h.getRegionInfo();
3428 
3429             MetaEntry m = null;
3430             if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
3431               m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
3432             } else {
3433               m = new MetaEntry(hri, sn, ts, null, null);
3434             }
3435             HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
3436             if (previous == null) {
3437               regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
3438             } else if (previous.metaEntry == null) {
3439               previous.metaEntry = m;
3440             } else {
3441               throw new IOException("Two entries in hbase:meta are same " + previous);
3442             }
3443           }
3444           PairOfSameType<HRegionInfo> mergeRegions = MetaTableAccessor.getMergeRegions(result);
3445           for (HRegionInfo mergeRegion : new HRegionInfo[] {
3446               mergeRegions.getFirst(), mergeRegions.getSecond() }) {
3447             if (mergeRegion != null) {
3448               // This region is already been merged
3449               HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
3450               hbInfo.setMerged(true);
3451             }
3452           }
3453 
3454           // show proof of progress to the user, once for every 100 records.
3455           if (countRecord % 100 == 0) {
3456             errors.progress();
3457           }
3458           countRecord++;
3459           return true;
3460         } catch (RuntimeException e) {
3461           LOG.error("Result=" + result);
3462           throw e;
3463         }
3464       }
3465     };
3466     if (!checkMetaOnly) {
3467       // Scan hbase:meta to pick up user regions
3468       MetaTableAccessor.fullScanRegions(connection, visitor);
3469     }
3470 
3471     errors.print("");
3472     return true;
3473   }
3474 
3475   /**
3476    * Stores the regioninfo entries scanned from META
3477    */
3478   static class MetaEntry extends HRegionInfo {
3479     ServerName regionServer;   // server hosting this region
3480     long modTime;          // timestamp of most recent modification metadata
3481     HRegionInfo splitA, splitB; //split daughters
3482 
3483     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3484       this(rinfo, regionServer, modTime, null, null);
3485     }
3486 
3487     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3488         HRegionInfo splitA, HRegionInfo splitB) {
3489       super(rinfo);
3490       this.regionServer = regionServer;
3491       this.modTime = modTime;
3492       this.splitA = splitA;
3493       this.splitB = splitB;
3494     }
3495 
3496     @Override
3497     public boolean equals(Object o) {
3498       boolean superEq = super.equals(o);
3499       if (!superEq) {
3500         return superEq;
3501       }
3502 
3503       MetaEntry me = (MetaEntry) o;
3504       if (!regionServer.equals(me.regionServer)) {
3505         return false;
3506       }
3507       return (modTime == me.modTime);
3508     }
3509 
3510     @Override
3511     public int hashCode() {
3512       int hash = Arrays.hashCode(getRegionName());
3513       hash ^= getRegionId();
3514       hash ^= Arrays.hashCode(getStartKey());
3515       hash ^= Arrays.hashCode(getEndKey());
3516       hash ^= Boolean.valueOf(isOffline()).hashCode();
3517       hash ^= getTable().hashCode();
3518       if (regionServer != null) {
3519         hash ^= regionServer.hashCode();
3520       }
3521       hash ^= modTime;
3522       return hash;
3523     }
3524   }
3525 
3526   /**
3527    * Stores the regioninfo entries from HDFS
3528    */
3529   static class HdfsEntry {
3530     HRegionInfo hri;
3531     Path hdfsRegionDir = null;
3532     long hdfsRegionDirModTime  = 0;
3533     boolean hdfsRegioninfoFilePresent = false;
3534     boolean hdfsOnlyEdits = false;
3535   }
3536 
3537   /**
3538    * Stores the regioninfo retrieved from Online region servers.
3539    */
3540   static class OnlineEntry {
3541     HRegionInfo hri;
3542     ServerName hsa;
3543 
3544     @Override
3545     public String toString() {
3546       return hsa.toString() + ";" + hri.getRegionNameAsString();
3547     }
3548   }
3549 
3550   /**
3551    * Maintain information about a particular region.  It gathers information
3552    * from three places -- HDFS, META, and region servers.
3553    */
3554   public static class HbckInfo implements KeyRange {
3555     private MetaEntry metaEntry = null; // info in META
3556     private HdfsEntry hdfsEntry = null; // info in HDFS
3557     private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
3558     private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
3559     private boolean skipChecks = false; // whether to skip further checks to this region info.
3560     private boolean isMerged = false;// whether this region has already been merged into another one
3561     private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3562     private HRegionInfo primaryHRIForDeployedReplica = null;
3563 
3564     HbckInfo(MetaEntry metaEntry) {
3565       this.metaEntry = metaEntry;
3566     }
3567 
3568     public synchronized int getReplicaId() {
3569       return metaEntry != null? metaEntry.getReplicaId(): deployedReplicaId;
3570     }
3571 
3572     public synchronized void addServer(HRegionInfo hri, ServerName server) {
3573       OnlineEntry rse = new OnlineEntry() ;
3574       rse.hri = hri;
3575       rse.hsa = server;
3576       this.deployedEntries.add(rse);
3577       this.deployedOn.add(server);
3578       // save the replicaId that we see deployed in the cluster
3579       this.deployedReplicaId = hri.getReplicaId();
3580       this.primaryHRIForDeployedReplica =
3581           RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3582     }
3583 
3584     @Override
3585     public synchronized String toString() {
3586       StringBuilder sb = new StringBuilder();
3587       sb.append("{ meta => ");
3588       sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3589       sb.append( ", hdfs => " + getHdfsRegionDir());
3590       sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3591       sb.append( ", replicaId => " + getReplicaId());
3592       sb.append(" }");
3593       return sb.toString();
3594     }
3595 
3596     @Override
3597     public byte[] getStartKey() {
3598       if (this.metaEntry != null) {
3599         return this.metaEntry.getStartKey();
3600       } else if (this.hdfsEntry != null) {
3601         return this.hdfsEntry.hri.getStartKey();
3602       } else {
3603         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3604         return null;
3605       }
3606     }
3607 
3608     @Override
3609     public byte[] getEndKey() {
3610       if (this.metaEntry != null) {
3611         return this.metaEntry.getEndKey();
3612       } else if (this.hdfsEntry != null) {
3613         return this.hdfsEntry.hri.getEndKey();
3614       } else {
3615         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3616         return null;
3617       }
3618     }
3619 
3620     public TableName getTableName() {
3621       if (this.metaEntry != null) {
3622         return this.metaEntry.getTable();
3623       } else if (this.hdfsEntry != null) {
3624         // we are only guaranteed to have a path and not an HRI for hdfsEntry,
3625         // so we get the name from the Path
3626         Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3627         return FSUtils.getTableName(tableDir);
3628       } else {
3629         // return the info from the first online/deployed hri
3630         for (OnlineEntry e : deployedEntries) {
3631           return e.hri.getTable();
3632         }
3633         return null;
3634       }
3635     }
3636 
3637     public String getRegionNameAsString() {
3638       if (metaEntry != null) {
3639         return metaEntry.getRegionNameAsString();
3640       } else if (hdfsEntry != null) {
3641         if (hdfsEntry.hri != null) {
3642           return hdfsEntry.hri.getRegionNameAsString();
3643         }
3644       } else {
3645         // return the info from the first online/deployed hri
3646         for (OnlineEntry e : deployedEntries) {
3647           return e.hri.getRegionNameAsString();
3648         }
3649       }
3650       return null;
3651     }
3652 
3653     public byte[] getRegionName() {
3654       if (metaEntry != null) {
3655         return metaEntry.getRegionName();
3656       } else if (hdfsEntry != null) {
3657         return hdfsEntry.hri.getRegionName();
3658       } else {
3659         // return the info from the first online/deployed hri
3660         for (OnlineEntry e : deployedEntries) {
3661           return e.hri.getRegionName();
3662         }
3663         return null;
3664       }
3665     }
3666 
3667     public HRegionInfo getPrimaryHRIForDeployedReplica() {
3668       return primaryHRIForDeployedReplica;
3669     }
3670 
3671     Path getHdfsRegionDir() {
3672       if (hdfsEntry == null) {
3673         return null;
3674       }
3675       return hdfsEntry.hdfsRegionDir;
3676     }
3677 
3678     boolean containsOnlyHdfsEdits() {
3679       if (hdfsEntry == null) {
3680         return false;
3681       }
3682       return hdfsEntry.hdfsOnlyEdits;
3683     }
3684 
3685     boolean isHdfsRegioninfoPresent() {
3686       if (hdfsEntry == null) {
3687         return false;
3688       }
3689       return hdfsEntry.hdfsRegioninfoFilePresent;
3690     }
3691 
3692     long getModTime() {
3693       if (hdfsEntry == null) {
3694         return 0;
3695       }
3696       return hdfsEntry.hdfsRegionDirModTime;
3697     }
3698 
3699     HRegionInfo getHdfsHRI() {
3700       if (hdfsEntry == null) {
3701         return null;
3702       }
3703       return hdfsEntry.hri;
3704     }
3705 
3706     public void setSkipChecks(boolean skipChecks) {
3707       this.skipChecks = skipChecks;
3708     }
3709 
3710     public boolean isSkipChecks() {
3711       return skipChecks;
3712     }
3713 
3714     public void setMerged(boolean isMerged) {
3715       this.isMerged = isMerged;
3716     }
3717 
3718     public boolean isMerged() {
3719       return this.isMerged;
3720     }
3721   }
3722 
3723   final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3724     @Override
3725     public int compare(HbckInfo l, HbckInfo r) {
3726       if (l == r) {
3727         // same instance
3728         return 0;
3729       }
3730 
3731       int tableCompare = l.getTableName().compareTo(r.getTableName());
3732       if (tableCompare != 0) {
3733         return tableCompare;
3734       }
3735 
3736       int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3737           l.getStartKey(), r.getStartKey());
3738       if (startComparison != 0) {
3739         return startComparison;
3740       }
3741 
3742       // Special case for absolute endkey
3743       byte[] endKey = r.getEndKey();
3744       endKey = (endKey.length == 0) ? null : endKey;
3745       byte[] endKey2 = l.getEndKey();
3746       endKey2 = (endKey2.length == 0) ? null : endKey2;
3747       int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3748           endKey2,  endKey);
3749 
3750       if (endComparison != 0) {
3751         return endComparison;
3752       }
3753 
3754       // use regionId as tiebreaker.
3755       // Null is considered after all possible values so make it bigger.
3756       if (l.hdfsEntry == null && r.hdfsEntry == null) {
3757         return 0;
3758       }
3759       if (l.hdfsEntry == null && r.hdfsEntry != null) {
3760         return 1;
3761       }
3762       // l.hdfsEntry must not be null
3763       if (r.hdfsEntry == null) {
3764         return -1;
3765       }
3766       // both l.hdfsEntry and r.hdfsEntry must not be null.
3767       return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3768     }
3769   };
3770 
3771   /**
3772    * Prints summary of all tables found on the system.
3773    */
3774   private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3775     StringBuilder sb = new StringBuilder();
3776     int numOfSkippedRegions;
3777     errors.print("Summary:");
3778     for (TableInfo tInfo : tablesInfo.values()) {
3779       numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName())) ?
3780           skippedRegions.get(tInfo.getName()).size() : 0;
3781 
3782       if (errors.tableHasErrors(tInfo)) {
3783         errors.print("Table " + tInfo.getName() + " is inconsistent.");
3784       } else if (numOfSkippedRegions > 0){
3785         errors.print("Table " + tInfo.getName() + " is okay (with "
3786           + numOfSkippedRegions + " skipped regions).");
3787       }
3788       else {
3789         errors.print("Table " + tInfo.getName() + " is okay.");
3790       }
3791       errors.print("    Number of regions: " + tInfo.getNumRegions());
3792       if (numOfSkippedRegions > 0) {
3793         Set<String> skippedRegionStrings = skippedRegions.get(tInfo.getName());
3794         System.out.println("    Number of skipped regions: " + numOfSkippedRegions);
3795         System.out.println("      List of skipped regions:");
3796         for(String sr : skippedRegionStrings) {
3797           System.out.println("        " + sr);
3798         }
3799       }
3800       sb.setLength(0); // clear out existing buffer, if any.
3801       sb.append("    Deployed on: ");
3802       for (ServerName server : tInfo.deployedOn) {
3803         sb.append(" " + server.toString());
3804       }
3805       errors.print(sb.toString());
3806     }
3807   }
3808 
3809   static ErrorReporter getErrorReporter(
3810       final Configuration conf) throws ClassNotFoundException {
3811     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3812     return ReflectionUtils.newInstance(reporter, conf);
3813   }
3814 
3815   public interface ErrorReporter {
3816     enum ERROR_CODE {
3817       UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3818       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META,
3819       NOT_DEPLOYED,
3820       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3821       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3822       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3823       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3824       WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR, ORPHAN_TABLE_STATE,
3825       NO_TABLE_STATE, UNDELETED_REPLICATION_QUEUE
3826     }
3827     void clear();
3828     void report(String message);
3829     void reportError(String message);
3830     void reportError(ERROR_CODE errorCode, String message);
3831     void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3832     void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3833     void reportError(
3834       ERROR_CODE errorCode,
3835       String message,
3836       TableInfo table,
3837       HbckInfo info1,
3838       HbckInfo info2
3839     );
3840     int summarize();
3841     void detail(String details);
3842     ArrayList<ERROR_CODE> getErrorList();
3843     void progress();
3844     void print(String message);
3845     void resetErrors();
3846     boolean tableHasErrors(TableInfo table);
3847   }
3848 
3849   static class PrintingErrorReporter implements ErrorReporter {
3850     public int errorCount = 0;
3851     private int showProgress;
3852     // How frequently calls to progress() will create output
3853     private static final int progressThreshold = 100;
3854 
3855     Set<TableInfo> errorTables = new HashSet<TableInfo>();
3856 
3857     // for use by unit tests to verify which errors were discovered
3858     private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3859 
3860     @Override
3861     public void clear() {
3862       errorTables.clear();
3863       errorList.clear();
3864       errorCount = 0;
3865     }
3866 
3867     @Override
3868     public synchronized void reportError(ERROR_CODE errorCode, String message) {
3869       if (errorCode == ERROR_CODE.WRONG_USAGE) {
3870         System.err.println(message);
3871         return;
3872       }
3873 
3874       errorList.add(errorCode);
3875       if (!summary) {
3876         System.out.println("ERROR: " + message);
3877       }
3878       errorCount++;
3879       showProgress = 0;
3880     }
3881 
3882     @Override
3883     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
3884       errorTables.add(table);
3885       reportError(errorCode, message);
3886     }
3887 
3888     @Override
3889     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3890                                          HbckInfo info) {
3891       errorTables.add(table);
3892       String reference = "(region " + info.getRegionNameAsString() + ")";
3893       reportError(errorCode, reference + " " + message);
3894     }
3895 
3896     @Override
3897     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3898                                          HbckInfo info1, HbckInfo info2) {
3899       errorTables.add(table);
3900       String reference = "(regions " + info1.getRegionNameAsString()
3901           + " and " + info2.getRegionNameAsString() + ")";
3902       reportError(errorCode, reference + " " + message);
3903     }
3904 
3905     @Override
3906     public synchronized void reportError(String message) {
3907       reportError(ERROR_CODE.UNKNOWN, message);
3908     }
3909 
3910     /**
3911      * Report error information, but do not increment the error count.  Intended for cases
3912      * where the actual error would have been reported previously.
3913      * @param message
3914      */
3915     @Override
3916     public synchronized void report(String message) {
3917       if (! summary) {
3918         System.out.println("ERROR: " + message);
3919       }
3920       showProgress = 0;
3921     }
3922 
3923     @Override
3924     public synchronized int summarize() {
3925       System.out.println(Integer.toString(errorCount) +
3926                          " inconsistencies detected.");
3927       if (errorCount == 0) {
3928         System.out.println("Status: OK");
3929         return 0;
3930       } else {
3931         System.out.println("Status: INCONSISTENT");
3932         return -1;
3933       }
3934     }
3935 
3936     @Override
3937     public ArrayList<ERROR_CODE> getErrorList() {
3938       return errorList;
3939     }
3940 
3941     @Override
3942     public synchronized void print(String message) {
3943       if (!summary) {
3944         System.out.println(message);
3945       }
3946     }
3947 
3948     @Override
3949     public boolean tableHasErrors(TableInfo table) {
3950       return errorTables.contains(table);
3951     }
3952 
3953     @Override
3954     public void resetErrors() {
3955       errorCount = 0;
3956     }
3957 
3958     @Override
3959     public synchronized void detail(String message) {
3960       if (details) {
3961         System.out.println(message);
3962       }
3963       showProgress = 0;
3964     }
3965 
3966     @Override
3967     public synchronized void progress() {
3968       if (showProgress++ == progressThreshold) {
3969         if (!summary) {
3970           System.out.print(".");
3971         }
3972         showProgress = 0;
3973       }
3974     }
3975   }
3976 
3977   /**
3978    * Contact a region server and get all information from it
3979    */
3980   static class WorkItemRegion implements Callable<Void> {
3981     private HBaseFsck hbck;
3982     private ServerName rsinfo;
3983     private ErrorReporter errors;
3984     private HConnection connection;
3985 
3986     WorkItemRegion(HBaseFsck hbck, ServerName info,
3987                    ErrorReporter errors, HConnection connection) {
3988       this.hbck = hbck;
3989       this.rsinfo = info;
3990       this.errors = errors;
3991       this.connection = connection;
3992     }
3993 
3994     @Override
3995     public synchronized Void call() throws IOException {
3996       errors.progress();
3997       try {
3998         BlockingInterface server = connection.getAdmin(rsinfo);
3999 
4000         // list all online regions from this region server
4001         List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
4002         regions = filterRegions(regions);
4003 
4004         if (details) {
4005           errors.detail("RegionServer: " + rsinfo.getServerName() +
4006                            " number of regions: " + regions.size());
4007           for (HRegionInfo rinfo: regions) {
4008             errors.detail("  " + rinfo.getRegionNameAsString() +
4009                              " id: " + rinfo.getRegionId() +
4010                              " encoded_name: " + rinfo.getEncodedName() +
4011                              " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
4012                              " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
4013           }
4014         }
4015 
4016         // check to see if the existence of this region matches the region in META
4017         for (HRegionInfo r:regions) {
4018           HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
4019           hbi.addServer(r, rsinfo);
4020         }
4021       } catch (IOException e) {          // unable to connect to the region server.
4022         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
4023           " Unable to fetch region information. " + e);
4024         throw e;
4025       }
4026       return null;
4027     }
4028 
4029     private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
4030       List<HRegionInfo> ret = Lists.newArrayList();
4031       for (HRegionInfo hri : regions) {
4032         if (hri.isMetaTable() || (!hbck.checkMetaOnly
4033             && hbck.isTableIncluded(hri.getTable()))) {
4034           ret.add(hri);
4035         }
4036       }
4037       return ret;
4038     }
4039   }
4040 
4041   /**
4042    * Contact hdfs and get all information about specified table directory into
4043    * regioninfo list.
4044    */
4045   static class WorkItemHdfsDir implements Callable<Void> {
4046     private HBaseFsck hbck;
4047     private FileStatus tableDir;
4048     private ErrorReporter errors;
4049     private FileSystem fs;
4050 
4051     WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
4052                     FileStatus status) {
4053       this.hbck = hbck;
4054       this.fs = fs;
4055       this.tableDir = status;
4056       this.errors = errors;
4057     }
4058 
4059     @Override
4060     public synchronized Void call() throws IOException {
4061       try {
4062         // level 2: <HBASE_DIR>/<table>/*
4063         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
4064         for (FileStatus regionDir : regionDirs) {
4065           errors.progress();
4066           String encodedName = regionDir.getPath().getName();
4067           // ignore directories that aren't hexadecimal
4068           if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
4069             continue;
4070           }
4071 
4072           LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
4073           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
4074           HdfsEntry he = new HdfsEntry();
4075           synchronized (hbi) {
4076             if (hbi.getHdfsRegionDir() != null) {
4077               errors.print("Directory " + encodedName + " duplicate??" +
4078                            hbi.getHdfsRegionDir());
4079             }
4080 
4081             he.hdfsRegionDir = regionDir.getPath();
4082             he.hdfsRegionDirModTime = regionDir.getModificationTime();
4083             Path regioninfoFile = new Path(he.hdfsRegionDir, HRegionFileSystem.REGION_INFO_FILE);
4084             he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
4085             // we add to orphan list when we attempt to read .regioninfo
4086 
4087             // Set a flag if this region contains only edits
4088             // This is special case if a region is left after split
4089             he.hdfsOnlyEdits = true;
4090             FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
4091             Path ePath = WALSplitter.getRegionDirRecoveredEditsDir(regionDir.getPath());
4092             for (FileStatus subDir : subDirs) {
4093               errors.progress();
4094               String sdName = subDir.getPath().getName();
4095               if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
4096                 he.hdfsOnlyEdits = false;
4097                 break;
4098               }
4099             }
4100             hbi.hdfsEntry = he;
4101           }
4102         }
4103       } catch (IOException e) {
4104         // unable to connect to the region server.
4105         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
4106             + tableDir.getPath().getName()
4107             + " Unable to fetch region information. " + e);
4108         throw e;
4109       }
4110       return null;
4111     }
4112   }
4113 
4114   /**
4115    * Contact hdfs and get all information about specified table directory into
4116    * regioninfo list.
4117    */
4118   static class WorkItemHdfsRegionInfo implements Callable<Void> {
4119     private HbckInfo hbi;
4120     private HBaseFsck hbck;
4121     private ErrorReporter errors;
4122 
4123     WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
4124       this.hbi = hbi;
4125       this.hbck = hbck;
4126       this.errors = errors;
4127     }
4128 
4129     @Override
4130     public synchronized Void call() throws IOException {
4131       // only load entries that haven't been loaded yet.
4132       if (hbi.getHdfsHRI() == null) {
4133         try {
4134           errors.progress();
4135           hbck.loadHdfsRegioninfo(hbi);
4136         } catch (IOException ioe) {
4137           String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
4138               + hbi.getTableName() + " in hdfs dir "
4139               + hbi.getHdfsRegionDir()
4140               + "!  It may be an invalid format or version file.  Treating as "
4141               + "an orphaned regiondir.";
4142           errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
4143           try {
4144             hbck.debugLsr(hbi.getHdfsRegionDir());
4145           } catch (IOException ioe2) {
4146             LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
4147             throw ioe2;
4148           }
4149           hbck.orphanHdfsDirs.add(hbi);
4150           throw ioe;
4151         }
4152       }
4153       return null;
4154     }
4155   };
4156 
4157   /**
4158    * Display the full report from fsck. This displays all live and dead region
4159    * servers, and all known regions.
4160    */
4161   public static void setDisplayFullReport() {
4162     details = true;
4163   }
4164 
4165   /**
4166    * Set exclusive mode.
4167    */
4168   public static void setForceExclusive() {
4169     forceExclusive = true;
4170   }
4171 
4172   /**
4173    * Only one instance of hbck can modify HBase at a time.
4174    */
4175   public boolean isExclusive() {
4176     return fixAny || forceExclusive;
4177   }
4178 
4179   /**
4180    * Disable the load balancer.
4181    */
4182   public static void setDisableBalancer() {
4183     disableBalancer = true;
4184   }
4185 
4186   /**
4187    * The balancer should be disabled if we are modifying HBase.
4188    * It can be disabled if you want to prevent region movement from causing
4189    * false positives.
4190    */
4191   public boolean shouldDisableBalancer() {
4192     return fixAny || disableBalancer;
4193   }
4194 
4195   /**
4196    * Set summary mode.
4197    * Print only summary of the tables and status (OK or INCONSISTENT)
4198    */
4199   static void setSummary() {
4200     summary = true;
4201   }
4202 
4203   /**
4204    * Set hbase:meta check mode.
4205    * Print only info about hbase:meta table deployment/state
4206    */
4207   void setCheckMetaOnly() {
4208     checkMetaOnly = true;
4209   }
4210 
4211   /**
4212    * Set region boundaries check mode.
4213    */
4214   void setRegionBoundariesCheck() {
4215     checkRegionBoundaries = true;
4216   }
4217 
4218   /**
4219    * Set table locks fix mode.
4220    * Delete table locks held for a long time
4221    */
4222   public void setFixTableLocks(boolean shouldFix) {
4223     fixTableLocks = shouldFix;
4224     fixAny |= shouldFix;
4225   }
4226   
4227   /**
4228    * Set replication fix mode.
4229    */
4230   public void setFixReplication(boolean shouldFix) {
4231     fixReplication = shouldFix;
4232     fixAny |= shouldFix;
4233   }
4234 
4235   /**
4236    * Check if we should rerun fsck again. This checks if we've tried to
4237    * fix something and we should rerun fsck tool again.
4238    * Display the full report from fsck. This displays all live and dead
4239    * region servers, and all known regions.
4240    */
4241   void setShouldRerun() {
4242     rerun = true;
4243   }
4244 
4245   boolean shouldRerun() {
4246     return rerun;
4247   }
4248 
4249   /**
4250    * Fix inconsistencies found by fsck. This should try to fix errors (if any)
4251    * found by fsck utility.
4252    */
4253   public void setFixAssignments(boolean shouldFix) {
4254     fixAssignments = shouldFix;
4255     fixAny |= shouldFix;
4256   }
4257 
4258   boolean shouldFixAssignments() {
4259     return fixAssignments;
4260   }
4261 
4262   public void setFixMeta(boolean shouldFix) {
4263     fixMeta = shouldFix;
4264     fixAny |= shouldFix;
4265   }
4266 
4267   boolean shouldFixMeta() {
4268     return fixMeta;
4269   }
4270 
4271   public void setFixEmptyMetaCells(boolean shouldFix) {
4272     fixEmptyMetaCells = shouldFix;
4273     fixAny |= shouldFix;
4274   }
4275 
4276   boolean shouldFixEmptyMetaCells() {
4277     return fixEmptyMetaCells;
4278   }
4279 
4280   public void setCheckHdfs(boolean checking) {
4281     checkHdfs = checking;
4282   }
4283 
4284   boolean shouldCheckHdfs() {
4285     return checkHdfs;
4286   }
4287 
4288   public void setFixHdfsHoles(boolean shouldFix) {
4289     fixHdfsHoles = shouldFix;
4290     fixAny |= shouldFix;
4291   }
4292 
4293   boolean shouldFixHdfsHoles() {
4294     return fixHdfsHoles;
4295   }
4296 
4297   public void setFixTableOrphans(boolean shouldFix) {
4298     fixTableOrphans = shouldFix;
4299     fixAny |= shouldFix;
4300   }
4301 
4302   boolean shouldFixTableOrphans() {
4303     return fixTableOrphans;
4304   }
4305 
4306   public void setFixHdfsOverlaps(boolean shouldFix) {
4307     fixHdfsOverlaps = shouldFix;
4308     fixAny |= shouldFix;
4309   }
4310 
4311   boolean shouldFixHdfsOverlaps() {
4312     return fixHdfsOverlaps;
4313   }
4314 
4315   public void setFixHdfsOrphans(boolean shouldFix) {
4316     fixHdfsOrphans = shouldFix;
4317     fixAny |= shouldFix;
4318   }
4319 
4320   boolean shouldFixHdfsOrphans() {
4321     return fixHdfsOrphans;
4322   }
4323 
4324   public void setFixVersionFile(boolean shouldFix) {
4325     fixVersionFile = shouldFix;
4326     fixAny |= shouldFix;
4327   }
4328 
4329   public boolean shouldFixVersionFile() {
4330     return fixVersionFile;
4331   }
4332 
4333   public void setSidelineBigOverlaps(boolean sbo) {
4334     this.sidelineBigOverlaps = sbo;
4335   }
4336 
4337   public boolean shouldSidelineBigOverlaps() {
4338     return sidelineBigOverlaps;
4339   }
4340 
4341   public void setFixSplitParents(boolean shouldFix) {
4342     fixSplitParents = shouldFix;
4343     fixAny |= shouldFix;
4344   }
4345 
4346   boolean shouldFixSplitParents() {
4347     return fixSplitParents;
4348   }
4349 
4350   public void setFixReferenceFiles(boolean shouldFix) {
4351     fixReferenceFiles = shouldFix;
4352     fixAny |= shouldFix;
4353   }
4354 
4355   boolean shouldFixReferenceFiles() {
4356     return fixReferenceFiles;
4357   }
4358 
4359   public boolean shouldIgnorePreCheckPermission() {
4360     return !fixAny || ignorePreCheckPermission;
4361   }
4362 
4363   public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
4364     this.ignorePreCheckPermission = ignorePreCheckPermission;
4365   }
4366 
4367   /**
4368    * @param mm maximum number of regions to merge into a single region.
4369    */
4370   public void setMaxMerge(int mm) {
4371     this.maxMerge = mm;
4372   }
4373 
4374   public int getMaxMerge() {
4375     return maxMerge;
4376   }
4377 
4378   public void setMaxOverlapsToSideline(int mo) {
4379     this.maxOverlapsToSideline = mo;
4380   }
4381 
4382   public int getMaxOverlapsToSideline() {
4383     return maxOverlapsToSideline;
4384   }
4385 
4386   /**
4387    * Only check/fix tables specified by the list,
4388    * Empty list means all tables are included.
4389    */
4390   boolean isTableIncluded(TableName table) {
4391     return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
4392   }
4393 
4394   public void includeTable(TableName table) {
4395     tablesIncluded.add(table);
4396   }
4397 
4398   Set<TableName> getIncludedTables() {
4399     return new HashSet<TableName>(tablesIncluded);
4400   }
4401 
4402   /**
4403    * We are interested in only those tables that have not changed their state in
4404    * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
4405    * @param seconds - the time in seconds
4406    */
4407   public void setTimeLag(long seconds) {
4408     timelag = seconds * 1000; // convert to milliseconds
4409   }
4410 
4411   /**
4412    *
4413    * @param sidelineDir - HDFS path to sideline data
4414    */
4415   public void setSidelineDir(String sidelineDir) {
4416     this.sidelineDir = new Path(sidelineDir);
4417   }
4418 
4419   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
4420     return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
4421   }
4422 
4423   public HFileCorruptionChecker getHFilecorruptionChecker() {
4424     return hfcc;
4425   }
4426 
4427   public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
4428     this.hfcc = hfcc;
4429   }
4430 
4431   public void setRetCode(int code) {
4432     this.retcode = code;
4433   }
4434 
4435   public int getRetCode() {
4436     return retcode;
4437   }
4438 
4439   protected HBaseFsck printUsageAndExit() {
4440     StringWriter sw = new StringWriter(2048);
4441     PrintWriter out = new PrintWriter(sw);
4442     out.println("Usage: fsck [opts] {only tables}");
4443     out.println(" where [opts] are:");
4444     out.println("   -help Display help options (this)");
4445     out.println("   -details Display full report of all regions.");
4446     out.println("   -timelag <timeInSeconds>  Process only regions that " +
4447                        " have not experienced any metadata updates in the last " +
4448                        " <timeInSeconds> seconds.");
4449     out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
4450         " before checking if the fix worked if run with -fix");
4451     out.println("   -summary Print only summary of the tables and status.");
4452     out.println("   -metaonly Only check the state of the hbase:meta table.");
4453     out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
4454     out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
4455     out.println("   -exclusive Abort if another hbck is exclusive or fixing.");
4456     out.println("   -disableBalancer Disable the load balancer.");
4457 
4458     out.println("");
4459     out.println("  Metadata Repair options: (expert features, use with caution!)");
4460     out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
4461     out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
4462     out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
4463     out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
4464         + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
4465     out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
4466     out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
4467     out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
4468     out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
4469     out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
4470     out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
4471     out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
4472     out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
4473     out.println("   -fixSplitParents  Try to force offline split parents to be online.");
4474     out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
4475     out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
4476     out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
4477         + " (empty REGIONINFO_QUALIFIER rows)");
4478 
4479     out.println("");
4480     out.println("  Datafile Repair options: (expert features, use with caution!)");
4481     out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
4482     out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
4483 
4484     out.println("");
4485     out.println("  Metadata Repair shortcuts");
4486     out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
4487         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles -fixTableLocks");
4488     out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
4489 
4490     out.println("");
4491     out.println("  Table lock options");
4492     out.println("   -fixTableLocks    Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
4493 
4494     out.println("");
4495     out.println(" Replication options");
4496     out.println("   -fixReplication   Deletes replication queues for removed peers");
4497     
4498     out.flush();
4499     errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
4500 
4501     setRetCode(-2);
4502     return this;
4503   }
4504 
4505   /**
4506    * Main program
4507    *
4508    * @param args
4509    * @throws Exception
4510    */
4511   public static void main(String[] args) throws Exception {
4512     // create a fsck object
4513     Configuration conf = HBaseConfiguration.create();
4514     Path hbasedir = FSUtils.getRootDir(conf);
4515     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
4516     FSUtils.setFsDefault(conf, new Path(defaultFs));
4517     int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
4518     System.exit(ret);
4519   }
4520 
4521   /**
4522    * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
4523    */
4524   static class HBaseFsckTool extends Configured implements Tool {
4525     HBaseFsckTool(Configuration conf) { super(conf); }
4526     @Override
4527     public int run(String[] args) throws Exception {
4528       HBaseFsck hbck = new HBaseFsck(getConf());
4529       hbck.exec(hbck.executor, args);
4530       hbck.close();
4531       return hbck.getRetCode();
4532     }
4533   };
4534 
4535 
4536   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
4537     ServiceException, InterruptedException {
4538     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
4539 
4540     boolean checkCorruptHFiles = false;
4541     boolean sidelineCorruptHFiles = false;
4542 
4543     // Process command-line args.
4544     for (int i = 0; i < args.length; i++) {
4545       String cmd = args[i];
4546       if (cmd.equals("-help") || cmd.equals("-h")) {
4547         return printUsageAndExit();
4548       } else if (cmd.equals("-details")) {
4549         setDisplayFullReport();
4550       } else if (cmd.equals("-exclusive")) {
4551         setForceExclusive();
4552       } else if (cmd.equals("-disableBalancer")) {
4553         setDisableBalancer();
4554       } else if (cmd.equals("-timelag")) {
4555         if (i == args.length - 1) {
4556           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4557           return printUsageAndExit();
4558         }
4559         try {
4560           long timelag = Long.parseLong(args[i+1]);
4561           setTimeLag(timelag);
4562         } catch (NumberFormatException e) {
4563           errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4564           return printUsageAndExit();
4565         }
4566         i++;
4567       } else if (cmd.equals("-sleepBeforeRerun")) {
4568         if (i == args.length - 1) {
4569           errors.reportError(ERROR_CODE.WRONG_USAGE,
4570             "HBaseFsck: -sleepBeforeRerun needs a value.");
4571           return printUsageAndExit();
4572         }
4573         try {
4574           sleepBeforeRerun = Long.parseLong(args[i+1]);
4575         } catch (NumberFormatException e) {
4576           errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4577           return printUsageAndExit();
4578         }
4579         i++;
4580       } else if (cmd.equals("-sidelineDir")) {
4581         if (i == args.length - 1) {
4582           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4583           return printUsageAndExit();
4584         }
4585         i++;
4586         setSidelineDir(args[i]);
4587       } else if (cmd.equals("-fix")) {
4588         errors.reportError(ERROR_CODE.WRONG_USAGE,
4589           "This option is deprecated, please use  -fixAssignments instead.");
4590         setFixAssignments(true);
4591       } else if (cmd.equals("-fixAssignments")) {
4592         setFixAssignments(true);
4593       } else if (cmd.equals("-fixMeta")) {
4594         setFixMeta(true);
4595       } else if (cmd.equals("-noHdfsChecking")) {
4596         setCheckHdfs(false);
4597       } else if (cmd.equals("-fixHdfsHoles")) {
4598         setFixHdfsHoles(true);
4599       } else if (cmd.equals("-fixHdfsOrphans")) {
4600         setFixHdfsOrphans(true);
4601       } else if (cmd.equals("-fixTableOrphans")) {
4602         setFixTableOrphans(true);
4603       } else if (cmd.equals("-fixHdfsOverlaps")) {
4604         setFixHdfsOverlaps(true);
4605       } else if (cmd.equals("-fixVersionFile")) {
4606         setFixVersionFile(true);
4607       } else if (cmd.equals("-sidelineBigOverlaps")) {
4608         setSidelineBigOverlaps(true);
4609       } else if (cmd.equals("-fixSplitParents")) {
4610         setFixSplitParents(true);
4611       } else if (cmd.equals("-ignorePreCheckPermission")) {
4612         setIgnorePreCheckPermission(true);
4613       } else if (cmd.equals("-checkCorruptHFiles")) {
4614         checkCorruptHFiles = true;
4615       } else if (cmd.equals("-sidelineCorruptHFiles")) {
4616         sidelineCorruptHFiles = true;
4617       } else if (cmd.equals("-fixReferenceFiles")) {
4618         setFixReferenceFiles(true);
4619       } else if (cmd.equals("-fixEmptyMetaCells")) {
4620         setFixEmptyMetaCells(true);
4621       } else if (cmd.equals("-repair")) {
4622         // this attempts to merge overlapping hdfs regions, needs testing
4623         // under load
4624         setFixHdfsHoles(true);
4625         setFixHdfsOrphans(true);
4626         setFixMeta(true);
4627         setFixAssignments(true);
4628         setFixHdfsOverlaps(true);
4629         setFixVersionFile(true);
4630         setSidelineBigOverlaps(true);
4631         setFixSplitParents(false);
4632         setCheckHdfs(true);
4633         setFixReferenceFiles(true);
4634         setFixTableLocks(true);
4635       } else if (cmd.equals("-repairHoles")) {
4636         // this will make all missing hdfs regions available but may lose data
4637         setFixHdfsHoles(true);
4638         setFixHdfsOrphans(false);
4639         setFixMeta(true);
4640         setFixAssignments(true);
4641         setFixHdfsOverlaps(false);
4642         setSidelineBigOverlaps(false);
4643         setFixSplitParents(false);
4644         setCheckHdfs(true);
4645       } else if (cmd.equals("-maxOverlapsToSideline")) {
4646         if (i == args.length - 1) {
4647           errors.reportError(ERROR_CODE.WRONG_USAGE,
4648             "-maxOverlapsToSideline needs a numeric value argument.");
4649           return printUsageAndExit();
4650         }
4651         try {
4652           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4653           setMaxOverlapsToSideline(maxOverlapsToSideline);
4654         } catch (NumberFormatException e) {
4655           errors.reportError(ERROR_CODE.WRONG_USAGE,
4656             "-maxOverlapsToSideline needs a numeric value argument.");
4657           return printUsageAndExit();
4658         }
4659         i++;
4660       } else if (cmd.equals("-maxMerge")) {
4661         if (i == args.length - 1) {
4662           errors.reportError(ERROR_CODE.WRONG_USAGE,
4663             "-maxMerge needs a numeric value argument.");
4664           return printUsageAndExit();
4665         }
4666         try {
4667           int maxMerge = Integer.parseInt(args[i+1]);
4668           setMaxMerge(maxMerge);
4669         } catch (NumberFormatException e) {
4670           errors.reportError(ERROR_CODE.WRONG_USAGE,
4671             "-maxMerge needs a numeric value argument.");
4672           return printUsageAndExit();
4673         }
4674         i++;
4675       } else if (cmd.equals("-summary")) {
4676         setSummary();
4677       } else if (cmd.equals("-metaonly")) {
4678         setCheckMetaOnly();
4679       } else if (cmd.equals("-boundaries")) {
4680         setRegionBoundariesCheck();
4681       } else if (cmd.equals("-fixTableLocks")) {
4682         setFixTableLocks(true);
4683       } else if (cmd.equals("-fixReplication")) {
4684         setFixReplication(true);
4685       } else if (cmd.startsWith("-")) {
4686         errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4687         return printUsageAndExit();
4688       } else {
4689         includeTable(TableName.valueOf(cmd));
4690         errors.print("Allow checking/fixes for table: " + cmd);
4691       }
4692     }
4693 
4694     errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
4695 
4696     // pre-check current user has FS write permission or not
4697     try {
4698       preCheckPermission();
4699     } catch (AccessDeniedException ace) {
4700       Runtime.getRuntime().exit(-1);
4701     } catch (IOException ioe) {
4702       Runtime.getRuntime().exit(-1);
4703     }
4704 
4705     // do the real work of hbck
4706     connect();
4707 
4708     try {
4709       // if corrupt file mode is on, first fix them since they may be opened later
4710       if (checkCorruptHFiles || sidelineCorruptHFiles) {
4711         LOG.info("Checking all hfiles for corruption");
4712         HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4713         setHFileCorruptionChecker(hfcc); // so we can get result
4714         Collection<TableName> tables = getIncludedTables();
4715         Collection<Path> tableDirs = new ArrayList<Path>();
4716         Path rootdir = FSUtils.getRootDir(getConf());
4717         if (tables.size() > 0) {
4718           for (TableName t : tables) {
4719             tableDirs.add(FSUtils.getTableDir(rootdir, t));
4720           }
4721         } else {
4722           tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4723         }
4724         hfcc.checkTables(tableDirs);
4725         hfcc.report(errors);
4726       }
4727 
4728       // check and fix table integrity, region consistency.
4729       int code = onlineHbck();
4730       setRetCode(code);
4731       // If we have changed the HBase state it is better to run hbck again
4732       // to see if we haven't broken something else in the process.
4733       // We run it only once more because otherwise we can easily fall into
4734       // an infinite loop.
4735       if (shouldRerun()) {
4736         try {
4737           LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4738           Thread.sleep(sleepBeforeRerun);
4739         } catch (InterruptedException ie) {
4740           LOG.warn("Interrupted while sleeping");
4741           return this;
4742         }
4743         // Just report
4744         setFixAssignments(false);
4745         setFixMeta(false);
4746         setFixHdfsHoles(false);
4747         setFixHdfsOverlaps(false);
4748         setFixVersionFile(false);
4749         setFixTableOrphans(false);
4750         errors.resetErrors();
4751         code = onlineHbck();
4752         setRetCode(code);
4753       }
4754     } finally {
4755       IOUtils.closeQuietly(this);
4756     }
4757     return this;
4758   }
4759 
4760   /**
4761    * ls -r for debugging purposes
4762    */
4763   void debugLsr(Path p) throws IOException {
4764     debugLsr(getConf(), p, errors);
4765   }
4766 
4767   /**
4768    * ls -r for debugging purposes
4769    */
4770   public static void debugLsr(Configuration conf,
4771       Path p) throws IOException {
4772     debugLsr(conf, p, new PrintingErrorReporter());
4773   }
4774 
4775   /**
4776    * ls -r for debugging purposes
4777    */
4778   public static void debugLsr(Configuration conf,
4779       Path p, ErrorReporter errors) throws IOException {
4780     if (!LOG.isDebugEnabled() || p == null) {
4781       return;
4782     }
4783     FileSystem fs = p.getFileSystem(conf);
4784 
4785     if (!fs.exists(p)) {
4786       // nothing
4787       return;
4788     }
4789     errors.print(p.toString());
4790 
4791     if (fs.isFile(p)) {
4792       return;
4793     }
4794 
4795     if (fs.getFileStatus(p).isDirectory()) {
4796       FileStatus[] fss= fs.listStatus(p);
4797       for (FileStatus status : fss) {
4798         debugLsr(conf, status.getPath(), errors);
4799       }
4800     }
4801   }
4802 }