View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.Closeable;
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.io.InterruptedIOException;
24  import java.io.PrintWriter;
25  import java.io.StringWriter;
26  import java.net.InetAddress;
27  import java.net.URI;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collection;
31  import java.util.Collections;
32  import java.util.Comparator;
33  import java.util.HashMap;
34  import java.util.HashSet;
35  import java.util.Iterator;
36  import java.util.List;
37  import java.util.Map;
38  import java.util.Map.Entry;
39  import java.util.Set;
40  import java.util.SortedMap;
41  import java.util.SortedSet;
42  import java.util.TreeMap;
43  import java.util.TreeSet;
44  import java.util.concurrent.Callable;
45  import java.util.concurrent.ConcurrentSkipListMap;
46  import java.util.concurrent.ExecutionException;
47  import java.util.concurrent.ExecutorService;
48  import java.util.concurrent.Executors;
49  import java.util.concurrent.Future;
50  import java.util.concurrent.FutureTask;
51  import java.util.concurrent.ScheduledThreadPoolExecutor;
52  import java.util.concurrent.TimeUnit;
53  import java.util.concurrent.TimeoutException;
54  import java.util.concurrent.atomic.AtomicBoolean;
55  import java.util.concurrent.atomic.AtomicInteger;
56  
57  import com.google.common.annotations.VisibleForTesting;
58  import com.google.common.base.Joiner;
59  import com.google.common.base.Preconditions;
60  import com.google.common.collect.ImmutableList;
61  import com.google.common.collect.Lists;
62  import com.google.common.collect.Multimap;
63  import com.google.common.collect.Ordering;
64  import com.google.common.collect.TreeMultimap;
65  import com.google.protobuf.ServiceException;
66  import org.apache.commons.io.IOUtils;
67  import org.apache.commons.lang.RandomStringUtils;
68  import org.apache.commons.lang.StringUtils;
69  import org.apache.commons.logging.Log;
70  import org.apache.commons.logging.LogFactory;
71  import org.apache.hadoop.conf.Configuration;
72  import org.apache.hadoop.conf.Configured;
73  import org.apache.hadoop.fs.FSDataOutputStream;
74  import org.apache.hadoop.fs.FileStatus;
75  import org.apache.hadoop.fs.FileSystem;
76  import org.apache.hadoop.fs.Path;
77  import org.apache.hadoop.fs.permission.FsAction;
78  import org.apache.hadoop.fs.permission.FsPermission;
79  import org.apache.hadoop.hbase.Abortable;
80  import org.apache.hadoop.hbase.Cell;
81  import org.apache.hadoop.hbase.CellUtil;
82  import org.apache.hadoop.hbase.ClusterStatus;
83  import org.apache.hadoop.hbase.HBaseConfiguration;
84  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
85  import org.apache.hadoop.hbase.HColumnDescriptor;
86  import org.apache.hadoop.hbase.HConstants;
87  import org.apache.hadoop.hbase.HRegionInfo;
88  import org.apache.hadoop.hbase.HRegionLocation;
89  import org.apache.hadoop.hbase.HTableDescriptor;
90  import org.apache.hadoop.hbase.KeyValue;
91  import org.apache.hadoop.hbase.MasterNotRunningException;
92  import org.apache.hadoop.hbase.MetaTableAccessor;
93  import org.apache.hadoop.hbase.RegionLocations;
94  import org.apache.hadoop.hbase.ServerName;
95  import org.apache.hadoop.hbase.TableDescriptor;
96  import org.apache.hadoop.hbase.TableName;
97  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
98  import org.apache.hadoop.hbase.classification.InterfaceAudience;
99  import org.apache.hadoop.hbase.classification.InterfaceStability;
100 import org.apache.hadoop.hbase.client.Admin;
101 import org.apache.hadoop.hbase.client.ClusterConnection;
102 import org.apache.hadoop.hbase.client.Connection;
103 import org.apache.hadoop.hbase.client.ConnectionFactory;
104 import org.apache.hadoop.hbase.client.Delete;
105 import org.apache.hadoop.hbase.client.Get;
106 import org.apache.hadoop.hbase.client.HConnection;
107 import org.apache.hadoop.hbase.client.Put;
108 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
109 import org.apache.hadoop.hbase.client.Result;
110 import org.apache.hadoop.hbase.client.RowMutations;
111 import org.apache.hadoop.hbase.client.Table;
112 import org.apache.hadoop.hbase.client.TableState;
113 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
114 import org.apache.hadoop.hbase.io.hfile.HFile;
115 import org.apache.hadoop.hbase.master.MasterFileSystem;
116 import org.apache.hadoop.hbase.master.RegionState;
117 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
118 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
119 import org.apache.hadoop.hbase.regionserver.HRegion;
120 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
121 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
122 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
123 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
124 import org.apache.hadoop.hbase.security.AccessDeniedException;
125 import org.apache.hadoop.hbase.security.UserProvider;
126 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
127 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
128 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
129 import org.apache.hadoop.hbase.util.hbck.ReplicationChecker;
130 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
131 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
132 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
133 import org.apache.hadoop.hbase.wal.WAL;
134 import org.apache.hadoop.hbase.wal.WALFactory;
135 import org.apache.hadoop.hbase.wal.WALSplitter;
136 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
137 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
138 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
139 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
140 import org.apache.hadoop.ipc.RemoteException;
141 import org.apache.hadoop.security.UserGroupInformation;
142 import org.apache.hadoop.util.ReflectionUtils;
143 import org.apache.hadoop.util.Tool;
144 import org.apache.hadoop.util.ToolRunner;
145 import org.apache.zookeeper.KeeperException;
146 
147 /**
148  * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
149  * table integrity problems in a corrupted HBase.
150  * <p>
151  * Region consistency checks verify that hbase:meta, region deployment on region
152  * servers and the state of data in HDFS (.regioninfo files) all are in
153  * accordance.
154  * <p>
155  * Table integrity checks verify that all possible row keys resolve to exactly
156  * one region of a table.  This means there are no individual degenerate
157  * or backwards regions; no holes between regions; and that there are no
158  * overlapping regions.
159  * <p>
160  * The general repair strategy works in two phases:
161  * <ol>
162  * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
163  * <li> Repair Region Consistency with hbase:meta and assignments
164  * </ol>
165  * <p>
166  * For table integrity repairs, the tables' region directories are scanned
167  * for .regioninfo files.  Each table's integrity is then verified.  If there
168  * are any orphan regions (regions with no .regioninfo files) or holes, new
169  * regions are fabricated.  Backwards regions are sidelined as well as empty
170  * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
171  * a new region is created and all data is merged into the new region.
172  * <p>
173  * Table integrity repairs deal solely with HDFS and could potentially be done
174  * offline -- the hbase region servers or master do not need to be running.
175  * This phase can eventually be used to completely reconstruct the hbase:meta table in
176  * an offline fashion.
177  * <p>
178  * Region consistency requires three conditions -- 1) valid .regioninfo file
179  * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
180  * and 3) a region is deployed only at the regionserver that was assigned to
181  * with proper state in the master.
182  * <p>
183  * Region consistency repairs require hbase to be online so that hbck can
184  * contact the HBase master and region servers.  The hbck#connect() method must
185  * first be called successfully.  Much of the region consistency information
186  * is transient and less risky to repair.
187  * <p>
188  * If hbck is run from the command line, there are a handful of arguments that
189  * can be used to limit the kinds of repairs hbck will do.  See the code in
190  * {@link #printUsageAndExit()} for more details.
191  */
192 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
193 @InterfaceStability.Evolving
194 public class HBaseFsck extends Configured implements Closeable {
195   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
196   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
197   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
198   private static boolean rsSupportsOffline = true;
199   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
200   private static final int DEFAULT_MAX_MERGE = 5;
201   private static final String TO_BE_LOADED = "to_be_loaded";
202   private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
203   private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5;
204   private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds
205   private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds
206   // We have to set the timeout value > HdfsConstants.LEASE_SOFTLIMIT_PERIOD.
207   // In HADOOP-2.6 and later, the Namenode proxy now created with custom RetryPolicy for
208   // AlreadyBeingCreatedException which is implies timeout on this operations up to
209   // HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds).
210   private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds
211 
212   /**********************
213    * Internal resources
214    **********************/
215   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
216   private ClusterStatus status;
217   private ClusterConnection connection;
218   private Admin admin;
219   private Table meta;
220   // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
221   protected ExecutorService executor;
222   private long startMillis = EnvironmentEdgeManager.currentTime();
223   private HFileCorruptionChecker hfcc;
224   private int retcode = 0;
225   private Path HBCK_LOCK_PATH;
226   private FSDataOutputStream hbckOutFd;
227   // This lock is to prevent cleanup of balancer resources twice between
228   // ShutdownHook and the main code. We cleanup only if the connect() is
229   // successful
230   private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
231 
232   /***********
233    * Options
234    ***********/
235   private static boolean details = false; // do we display the full report
236   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
237   private static boolean forceExclusive = false; // only this hbck can modify HBase
238   private static boolean disableBalancer = false; // disable load balancer to keep regions stable
239   private static boolean disableSplitAndMerge = false; // disable split and merge
240   private boolean fixAssignments = false; // fix assignment errors?
241   private boolean fixMeta = false; // fix meta errors?
242   private boolean checkHdfs = true; // load and check fs consistency?
243   private boolean fixHdfsHoles = false; // fix fs holes?
244   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
245   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
246   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
247   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
248   private boolean fixSplitParents = false; // fix lingering split parents
249   private boolean fixReferenceFiles = false; // fix lingering reference store file
250   private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
251   private boolean fixTableLocks = false; // fix table locks which are expired
252   private boolean fixReplication = false; // fix undeleted replication queues for removed peer
253   private boolean fixAny = false; // Set to true if any of the fix is required.
254 
255   // limit checking/fixes to listed tables, if empty attempt to check/fix all
256   // hbase:meta are always checked
257   private Set<TableName> tablesIncluded = new HashSet<TableName>();
258   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
259   // maximum number of overlapping regions to sideline
260   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE;
261   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
262   private Path sidelineDir = null;
263 
264   private boolean rerun = false; // if we tried to fix something, rerun hbck
265   private static boolean summary = false; // if we want to print less output
266   private boolean checkMetaOnly = false;
267   private boolean checkRegionBoundaries = false;
268   private boolean ignorePreCheckPermission = false; // if pre-check permission
269 
270   /*********
271    * State
272    *********/
273   final private ErrorReporter errors;
274   int fixes = 0;
275 
276   /**
277    * This map contains the state of all hbck items.  It maps from encoded region
278    * name to HbckInfo structure.  The information contained in HbckInfo is used
279    * to detect and correct consistency (hdfs/meta/deployment) problems.
280    */
281   private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
282   // Empty regioninfo qualifiers in hbase:meta
283   private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
284 
285   /**
286    * This map from Tablename -> TableInfo contains the structures necessary to
287    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
288    * to prevent dupes.
289    *
290    * If tablesIncluded is empty, this map contains all tables.
291    * Otherwise, it contains only meta tables and tables in tablesIncluded,
292    * unless checkMetaOnly is specified, in which case, it contains only
293    * the meta table
294    */
295   private SortedMap<TableName, TableInfo> tablesInfo =
296       new ConcurrentSkipListMap<TableName, TableInfo>();
297 
298   /**
299    * When initially looking at HDFS, we attempt to find any orphaned data.
300    */
301   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
302 
303   private Map<TableName, Set<String>> orphanTableDirs =
304       new HashMap<TableName, Set<String>>();
305   private Map<TableName, TableState> tableStates =
306       new HashMap<TableName, TableState>();
307   private final RetryCounterFactory lockFileRetryCounterFactory;
308 
309   private Map<TableName, Set<String>> skippedRegions = new HashMap<TableName, Set<String>>();
310 
311   ZooKeeperWatcher zkw = null;
312   /**
313    * Constructor
314    *
315    * @param conf Configuration object
316    * @throws MasterNotRunningException if the master is not running
317    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
318    */
319   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
320       ZooKeeperConnectionException, IOException, ClassNotFoundException {
321     this(conf, createThreadPool(conf));
322   }
323 
324   private static ExecutorService createThreadPool(Configuration conf) {
325     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
326     return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
327   }
328 
329   /**
330    * Constructor
331    *
332    * @param conf
333    *          Configuration object
334    * @throws MasterNotRunningException
335    *           if the master is not running
336    * @throws ZooKeeperConnectionException
337    *           if unable to connect to ZooKeeper
338    */
339   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
340       ZooKeeperConnectionException, IOException, ClassNotFoundException {
341     super(conf);
342     errors = getErrorReporter(getConf());
343     this.executor = exec;
344     lockFileRetryCounterFactory = new RetryCounterFactory(
345       getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
346       getConf().getInt(
347         "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),
348       getConf().getInt(
349         "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));
350     zkw = createZooKeeperWatcher();
351   }
352 
353   private class FileLockCallable implements Callable<FSDataOutputStream> {
354     RetryCounter retryCounter;
355 
356     public FileLockCallable(RetryCounter retryCounter) {
357       this.retryCounter = retryCounter;
358     }
359     @Override
360     public FSDataOutputStream call() throws IOException {
361       try {
362         FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
363         FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
364             HConstants.DATA_FILE_UMASK_KEY);
365         Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
366         fs.mkdirs(tmpDir);
367         HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
368         final FSDataOutputStream out = createFileWithRetries(fs, HBCK_LOCK_PATH, defaultPerms);
369         out.writeBytes(InetAddress.getLocalHost().toString());
370         out.flush();
371         return out;
372       } catch(RemoteException e) {
373         if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
374           return null;
375         } else {
376           throw e;
377         }
378       }
379     }
380 
381     private FSDataOutputStream createFileWithRetries(final FileSystem fs,
382         final Path hbckLockFilePath, final FsPermission defaultPerms)
383         throws IOException {
384 
385       IOException exception = null;
386       do {
387         try {
388           return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false);
389         } catch (IOException ioe) {
390           LOG.info("Failed to create lock file " + hbckLockFilePath.getName()
391               + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of "
392               + retryCounter.getMaxAttempts());
393           LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(),
394               ioe);
395           try {
396             exception = ioe;
397             retryCounter.sleepUntilNextRetry();
398           } catch (InterruptedException ie) {
399             throw (InterruptedIOException) new InterruptedIOException(
400                 "Can't create lock file " + hbckLockFilePath.getName())
401             .initCause(ie);
402           }
403         }
404       } while (retryCounter.shouldRetry());
405 
406       throw exception;
407     }
408   }
409 
410   /**
411    * This method maintains a lock using a file. If the creation fails we return null
412    *
413    * @return FSDataOutputStream object corresponding to the newly opened lock file
414    * @throws IOException
415    */
416   private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
417     RetryCounter retryCounter = lockFileRetryCounterFactory.create();
418     FileLockCallable callable = new FileLockCallable(retryCounter);
419     ExecutorService executor = Executors.newFixedThreadPool(1);
420     FutureTask<FSDataOutputStream> futureTask = new FutureTask<FSDataOutputStream>(callable);
421     executor.execute(futureTask);
422     final int timeoutInSeconds = getConf().getInt(
423       "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT);
424     FSDataOutputStream stream = null;
425     try {
426       stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);
427     } catch (ExecutionException ee) {
428       LOG.warn("Encountered exception when opening lock file", ee);
429     } catch (InterruptedException ie) {
430       LOG.warn("Interrupted when opening lock file", ie);
431       Thread.currentThread().interrupt();
432     } catch (TimeoutException exception) {
433       // took too long to obtain lock
434       LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");
435       futureTask.cancel(true);
436     } finally {
437       executor.shutdownNow();
438     }
439     return stream;
440   }
441 
442   private void unlockHbck() {
443     if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) {
444       RetryCounter retryCounter = lockFileRetryCounterFactory.create();
445       do {
446         try {
447           IOUtils.closeQuietly(hbckOutFd);
448           FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()),
449               HBCK_LOCK_PATH, true);
450           LOG.info("Finishing hbck");
451           return;
452         } catch (IOException ioe) {
453           LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try="
454               + (retryCounter.getAttemptTimes() + 1) + " of "
455               + retryCounter.getMaxAttempts());
456           LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe);
457           try {
458             retryCounter.sleepUntilNextRetry();
459           } catch (InterruptedException ie) {
460             Thread.currentThread().interrupt();
461             LOG.warn("Interrupted while deleting lock file" +
462                 HBCK_LOCK_PATH);
463             return;
464           }
465         }
466       } while (retryCounter.shouldRetry());
467     }
468   }
469 
470   /**
471    * To repair region consistency, one must call connect() in order to repair
472    * online state.
473    */
474   public void connect() throws IOException {
475 
476     if (isExclusive()) {
477       // Grab the lock
478       hbckOutFd = checkAndMarkRunningHbck();
479       if (hbckOutFd == null) {
480         setRetCode(-1);
481         LOG.error("Another instance of hbck is fixing HBase, exiting this instance. " +
482             "[If you are sure no other instance is running, delete the lock file " +
483             HBCK_LOCK_PATH + " and rerun the tool]");
484         throw new IOException("Duplicate hbck - Abort");
485       }
486 
487       // Make sure to cleanup the lock
488       hbckLockCleanup.set(true);
489     }
490 
491 
492     // Add a shutdown hook to this thread, in case user tries to
493     // kill the hbck with a ctrl-c, we want to cleanup the lock so that
494     // it is available for further calls
495     Runtime.getRuntime().addShutdownHook(new Thread() {
496       @Override
497       public void run() {
498         IOUtils.closeQuietly(HBaseFsck.this);
499         unlockHbck();
500       }
501     });
502 
503     LOG.info("Launching hbck");
504 
505     connection = (ClusterConnection)ConnectionFactory.createConnection(getConf());
506     admin = connection.getAdmin();
507     meta = connection.getTable(TableName.META_TABLE_NAME);
508     status = admin.getClusterStatus();
509   }
510 
511   /**
512    * Get deployed regions according to the region servers.
513    */
514   private void loadDeployedRegions() throws IOException, InterruptedException {
515     // From the master, get a list of all known live region servers
516     Collection<ServerName> regionServers = status.getServers();
517     errors.print("Number of live region servers: " + regionServers.size());
518     if (details) {
519       for (ServerName rsinfo: regionServers) {
520         errors.print("  " + rsinfo.getServerName());
521       }
522     }
523 
524     // From the master, get a list of all dead region servers
525     Collection<ServerName> deadRegionServers = status.getDeadServerNames();
526     errors.print("Number of dead region servers: " + deadRegionServers.size());
527     if (details) {
528       for (ServerName name: deadRegionServers) {
529         errors.print("  " + name);
530       }
531     }
532 
533     // Print the current master name and state
534     errors.print("Master: " + status.getMaster());
535 
536     // Print the list of all backup masters
537     Collection<ServerName> backupMasters = status.getBackupMasters();
538     errors.print("Number of backup masters: " + backupMasters.size());
539     if (details) {
540       for (ServerName name: backupMasters) {
541         errors.print("  " + name);
542       }
543     }
544 
545     errors.print("Average load: " + status.getAverageLoad());
546     errors.print("Number of requests: " + status.getRequestsCount());
547     errors.print("Number of regions: " + status.getRegionsCount());
548 
549     Map<String, RegionState> rits = status.getRegionsInTransition();
550     errors.print("Number of regions in transition: " + rits.size());
551     if (details) {
552       for (RegionState state: rits.values()) {
553         errors.print("  " + state.toDescriptiveString());
554       }
555     }
556 
557     // Determine what's deployed
558     processRegionServers(regionServers);
559   }
560 
561   /**
562    * Clear the current state of hbck.
563    */
564   private void clearState() {
565     // Make sure regionInfo is empty before starting
566     fixes = 0;
567     regionInfoMap.clear();
568     emptyRegionInfoQualifiers.clear();
569     tableStates.clear();
570     errors.clear();
571     tablesInfo.clear();
572     orphanHdfsDirs.clear();
573     skippedRegions.clear();
574   }
575 
576   /**
577    * This repair method analyzes hbase data in hdfs and repairs it to satisfy
578    * the table integrity rules.  HBase doesn't need to be online for this
579    * operation to work.
580    */
581   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
582     // Initial pass to fix orphans.
583     if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
584         || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
585       LOG.info("Loading regioninfos HDFS");
586       // if nothing is happening this should always complete in two iterations.
587       int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
588       int curIter = 0;
589       do {
590         clearState(); // clears hbck state and reset fixes to 0 and.
591         // repair what's on HDFS
592         restoreHdfsIntegrity();
593         curIter++;// limit the number of iterations.
594       } while (fixes > 0 && curIter <= maxIterations);
595 
596       // Repairs should be done in the first iteration and verification in the second.
597       // If there are more than 2 passes, something funny has happened.
598       if (curIter > 2) {
599         if (curIter == maxIterations) {
600           LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
601               + "Tables integrity may not be fully repaired!");
602         } else {
603           LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
604         }
605       }
606     }
607   }
608 
609   /**
610    * This repair method requires the cluster to be online since it contacts
611    * region servers and the masters.  It makes each region's state in HDFS, in
612    * hbase:meta, and deployments consistent.
613    *
614    * @return If &gt; 0 , number of errors detected, if &lt; 0 there was an unrecoverable
615    *     error.  If 0, we have a clean hbase.
616    */
617   public int onlineConsistencyRepair() throws IOException, KeeperException,
618     InterruptedException {
619     clearState();
620 
621     // get regions according to what is online on each RegionServer
622     loadDeployedRegions();
623     // check whether hbase:meta is deployed and online
624     recordMetaRegion();
625     // Check if hbase:meta is found only once and in the right place
626     if (!checkMetaRegion()) {
627       String errorMsg = "hbase:meta table is not consistent. ";
628       if (shouldFixAssignments()) {
629         errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
630       } else {
631         errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
632       }
633       errors.reportError(errorMsg + " Exiting...");
634       return -2;
635     }
636     // Not going with further consistency check for tables when hbase:meta itself is not consistent.
637     LOG.info("Loading regionsinfo from the hbase:meta table");
638     boolean success = loadMetaEntries();
639     if (!success) return -1;
640 
641     // Empty cells in hbase:meta?
642     reportEmptyMetaCells();
643 
644     // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
645     if (shouldFixEmptyMetaCells()) {
646       fixEmptyMetaCells();
647     }
648 
649     // get a list of all tables that have not changed recently.
650     if (!checkMetaOnly) {
651       reportTablesInFlux();
652     }
653 
654     // Get disabled tables states
655     loadTableStates();
656 
657     // load regiondirs and regioninfos from HDFS
658     if (shouldCheckHdfs()) {
659       LOG.info("Loading region directories from HDFS");
660       loadHdfsRegionDirs();
661       LOG.info("Loading region information from HDFS");
662       loadHdfsRegionInfos();
663     }
664 
665     // fix the orphan tables
666     fixOrphanTables();
667 
668     LOG.info("Checking and fixing region consistency");
669     // Check and fix consistency
670     checkAndFixConsistency();
671 
672     // Check integrity (does not fix)
673     checkIntegrity();
674     return errors.getErrorList().size();
675   }
676 
677   /**
678    * Contacts the master and prints out cluster-wide information
679    * @return 0 on success, non-zero on failure
680    */
681   public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
682     // print hbase server version
683     errors.print("Version: " + status.getHBaseVersion());
684     offlineHdfsIntegrityRepair();
685 
686     boolean oldBalancer = false;
687     if (shouldDisableBalancer()) {
688       oldBalancer = admin.setBalancerRunning(false, true);
689     }
690     boolean[] oldSplitAndMerge = null;
691     if (shouldDisableSplitAndMerge()) {
692       admin.releaseSplitOrMergeLockAndRollback();
693       oldSplitAndMerge = admin.setSplitOrMergeEnabled(false, false, false,
694         Admin.MasterSwitchType.SPLIT, Admin.MasterSwitchType.MERGE);
695     }
696 
697     try {
698       onlineConsistencyRepair();
699     }
700     finally {
701       // Only restore the balancer if it was true when we started repairing and
702       // we actually disabled it. Otherwise, we might clobber another run of
703       // hbck that has just restored it.
704       if (shouldDisableBalancer() && oldBalancer) {
705         admin.setBalancerRunning(oldBalancer, false);
706       }
707 
708       if (shouldDisableSplitAndMerge()) {
709         if (oldSplitAndMerge != null) {
710           admin.releaseSplitOrMergeLockAndRollback();
711         }
712       }
713     }
714 
715     if (checkRegionBoundaries) {
716       checkRegionBoundaries();
717     }
718 
719     offlineReferenceFileRepair();
720 
721     checkAndFixTableLocks();
722 
723     checkAndFixReplication();
724     
725     // Remove the hbck lock
726     unlockHbck();
727 
728     // Print table summary
729     printTableSummary(tablesInfo);
730     return errors.summarize();
731   }
732 
733   public static byte[] keyOnly (byte[] b) {
734     if (b == null)
735       return b;
736     int rowlength = Bytes.toShort(b, 0);
737     byte[] result = new byte[rowlength];
738     System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
739     return result;
740   }
741 
742   @Override
743   public void close() throws IOException {
744     try {
745       unlockHbck();
746     } catch (Exception io) {
747       LOG.warn(io);
748     } finally {
749       if (zkw != null) {
750         zkw.close();
751         zkw = null;
752       }
753       IOUtils.closeQuietly(admin);
754       IOUtils.closeQuietly(meta);
755       IOUtils.closeQuietly(connection);
756     }
757   }
758 
759   private static class RegionBoundariesInformation {
760     public byte [] regionName;
761     public byte [] metaFirstKey;
762     public byte [] metaLastKey;
763     public byte [] storesFirstKey;
764     public byte [] storesLastKey;
765     @Override
766     public String toString () {
767       return "regionName=" + Bytes.toStringBinary(regionName) +
768              "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
769              "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
770              "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
771              "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
772     }
773   }
774 
775   public void checkRegionBoundaries() {
776     try {
777       ByteArrayComparator comparator = new ByteArrayComparator();
778       List<HRegionInfo> regions = MetaTableAccessor.getAllRegions(connection, true);
779       final RegionBoundariesInformation currentRegionBoundariesInformation =
780           new RegionBoundariesInformation();
781       Path hbaseRoot = FSUtils.getRootDir(getConf());
782       for (HRegionInfo regionInfo : regions) {
783         Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
784         currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
785         // For each region, get the start and stop key from the META and compare them to the
786         // same information from the Stores.
787         Path path = new Path(tableDir, regionInfo.getEncodedName());
788         FileSystem fs = path.getFileSystem(getConf());
789         FileStatus[] files = fs.listStatus(path);
790         // For all the column families in this region...
791         byte[] storeFirstKey = null;
792         byte[] storeLastKey = null;
793         for (FileStatus file : files) {
794           String fileName = file.getPath().toString();
795           fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
796           if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
797             FileStatus[] storeFiles = fs.listStatus(file.getPath());
798             // For all the stores in this column family.
799             for (FileStatus storeFile : storeFiles) {
800               HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
801                   getConf()), getConf());
802               if ((reader.getFirstKey() != null)
803                   && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
804                       ((KeyValue.KeyOnlyKeyValue) reader.getFirstKey()).getKey()) > 0))) {
805                 storeFirstKey = ((KeyValue.KeyOnlyKeyValue)reader.getFirstKey()).getKey();
806               }
807               if ((reader.getLastKey() != null)
808                   && ((storeLastKey == null) || (comparator.compare(storeLastKey,
809                       ((KeyValue.KeyOnlyKeyValue)reader.getLastKey()).getKey())) < 0)) {
810                 storeLastKey = ((KeyValue.KeyOnlyKeyValue)reader.getLastKey()).getKey();
811               }
812               reader.close();
813             }
814           }
815         }
816         currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
817         currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
818         currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
819         currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
820         if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
821           currentRegionBoundariesInformation.metaFirstKey = null;
822         if (currentRegionBoundariesInformation.metaLastKey.length == 0)
823           currentRegionBoundariesInformation.metaLastKey = null;
824 
825         // For a region to be correct, we need the META start key to be smaller or equal to the
826         // smallest start key from all the stores, and the start key from the next META entry to
827         // be bigger than the last key from all the current stores. First region start key is null;
828         // Last region end key is null; some regions can be empty and not have any store.
829 
830         boolean valid = true;
831         // Checking start key.
832         if ((currentRegionBoundariesInformation.storesFirstKey != null)
833             && (currentRegionBoundariesInformation.metaFirstKey != null)) {
834           valid = valid
835               && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
836                 currentRegionBoundariesInformation.metaFirstKey) >= 0;
837         }
838         // Checking stop key.
839         if ((currentRegionBoundariesInformation.storesLastKey != null)
840             && (currentRegionBoundariesInformation.metaLastKey != null)) {
841           valid = valid
842               && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
843                 currentRegionBoundariesInformation.metaLastKey) < 0;
844         }
845         if (!valid) {
846           errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
847             tablesInfo.get(regionInfo.getTable()));
848           LOG.warn("Region's boundaries not alligned between stores and META for:");
849           LOG.warn(currentRegionBoundariesInformation);
850         }
851       }
852     } catch (IOException e) {
853       LOG.error(e);
854     }
855   }
856 
857   /**
858    * Iterates through the list of all orphan/invalid regiondirs.
859    */
860   private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
861     for (HbckInfo hi : orphanHdfsDirs) {
862       LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
863       adoptHdfsOrphan(hi);
864     }
865   }
866 
867   /**
868    * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
869    * these orphans by creating a new region, and moving the column families,
870    * recovered edits, WALs, into the new region dir.  We determine the region
871    * startkey and endkeys by looking at all of the hfiles inside the column
872    * families to identify the min and max keys. The resulting region will
873    * likely violate table integrity but will be dealt with by merging
874    * overlapping regions.
875    */
876   @SuppressWarnings("deprecation")
877   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
878     Path p = hi.getHdfsRegionDir();
879     FileSystem fs = p.getFileSystem(getConf());
880     FileStatus[] dirs = fs.listStatus(p);
881     if (dirs == null) {
882       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
883           p + ". This dir could probably be deleted.");
884       return ;
885     }
886 
887     TableName tableName = hi.getTableName();
888     TableInfo tableInfo = tablesInfo.get(tableName);
889     Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
890     HTableDescriptor template = tableInfo.getHTD();
891 
892     // find min and max key values
893     Pair<byte[],byte[]> orphanRegionRange = null;
894     for (FileStatus cf : dirs) {
895       String cfName= cf.getPath().getName();
896       // TODO Figure out what the special dirs are
897       if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
898 
899       FileStatus[] hfiles = fs.listStatus(cf.getPath());
900       for (FileStatus hfile : hfiles) {
901         byte[] start, end;
902         HFile.Reader hf = null;
903         try {
904           CacheConfig cacheConf = new CacheConfig(getConf());
905           hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
906           hf.loadFileInfo();
907           Cell startKv = hf.getFirstKey();
908           start = CellUtil.cloneRow(startKv);
909           Cell endKv = hf.getLastKey();
910           end = CellUtil.cloneRow(endKv);
911         } catch (IOException ioe) {
912           LOG.warn("Problem reading orphan file " + hfile + ", skipping");
913           continue;
914         } catch (NullPointerException ioe) {
915           LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
916           continue;
917         } finally {
918           if (hf != null) {
919             hf.close();
920           }
921         }
922 
923         // expand the range to include the range of all hfiles
924         if (orphanRegionRange == null) {
925           // first range
926           orphanRegionRange = new Pair<byte[], byte[]>(start, end);
927         } else {
928           // TODO add test
929 
930           // expand range only if the hfile is wider.
931           if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
932             orphanRegionRange.setFirst(start);
933           }
934           if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
935             orphanRegionRange.setSecond(end);
936           }
937         }
938       }
939     }
940     if (orphanRegionRange == null) {
941       LOG.warn("No data in dir " + p + ", sidelining data");
942       fixes++;
943       sidelineRegionDir(fs, hi);
944       return;
945     }
946     LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
947         Bytes.toString(orphanRegionRange.getSecond()) + ")");
948 
949     // create new region on hdfs. move data into place.
950     HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(),
951         Bytes.add(orphanRegionRange.getSecond(), new byte[1]));
952     LOG.info("Creating new region : " + hri);
953     HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
954     Path target = region.getRegionFileSystem().getRegionDir();
955 
956     // rename all the data to new region
957     mergeRegionDirs(target, hi);
958     fixes++;
959   }
960 
961   /**
962    * This method determines if there are table integrity errors in HDFS.  If
963    * there are errors and the appropriate "fix" options are enabled, the method
964    * will first correct orphan regions making them into legit regiondirs, and
965    * then reload to merge potentially overlapping regions.
966    *
967    * @return number of table integrity errors found
968    */
969   private int restoreHdfsIntegrity() throws IOException, InterruptedException {
970     // Determine what's on HDFS
971     LOG.info("Loading HBase regioninfo from HDFS...");
972     loadHdfsRegionDirs(); // populating regioninfo table.
973 
974     int errs = errors.getErrorList().size();
975     // First time just get suggestions.
976     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
977     checkHdfsIntegrity(false, false);
978 
979     if (errors.getErrorList().size() == errs) {
980       LOG.info("No integrity errors.  We are done with this phase. Glorious.");
981       return 0;
982     }
983 
984     if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
985       adoptHdfsOrphans(orphanHdfsDirs);
986       // TODO optimize by incrementally adding instead of reloading.
987     }
988 
989     // Make sure there are no holes now.
990     if (shouldFixHdfsHoles()) {
991       clearState(); // this also resets # fixes.
992       loadHdfsRegionDirs();
993       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
994       tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
995     }
996 
997     // Now we fix overlaps
998     if (shouldFixHdfsOverlaps()) {
999       // second pass we fix overlaps.
1000       clearState(); // this also resets # fixes.
1001       loadHdfsRegionDirs();
1002       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1003       tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
1004     }
1005 
1006     return errors.getErrorList().size();
1007   }
1008 
1009   /**
1010    * Scan all the store file names to find any lingering reference files,
1011    * which refer to some none-exiting files. If "fix" option is enabled,
1012    * any lingering reference file will be sidelined if found.
1013    * <p>
1014    * Lingering reference file prevents a region from opening. It has to
1015    * be fixed before a cluster can start properly.
1016    */
1017   private void offlineReferenceFileRepair() throws IOException {
1018     Configuration conf = getConf();
1019     Path hbaseRoot = FSUtils.getRootDir(conf);
1020     FileSystem fs = hbaseRoot.getFileSystem(conf);
1021     LOG.info("Computing mapping of all store files");
1022     Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot, errors);
1023     errors.print("");
1024     LOG.info("Validating mapping using HDFS state");
1025     for (Path path: allFiles.values()) {
1026       boolean isReference = false;
1027       try {
1028         isReference = StoreFileInfo.isReference(path);
1029       } catch (Throwable t) {
1030         // Ignore. Some files may not be store files at all.
1031         // For example, files under .oldlogs folder in hbase:meta
1032         // Warning message is already logged by
1033         // StoreFile#isReference.
1034       }
1035       if (!isReference) continue;
1036 
1037       Path referredToFile = StoreFileInfo.getReferredToFile(path);
1038       if (fs.exists(referredToFile)) continue;  // good, expected
1039 
1040       // Found a lingering reference file
1041       errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
1042         "Found lingering reference file " + path);
1043       if (!shouldFixReferenceFiles()) continue;
1044 
1045       // Now, trying to fix it since requested
1046       boolean success = false;
1047       String pathStr = path.toString();
1048 
1049       // A reference file path should be like
1050       // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
1051       // Up 5 directories to get the root folder.
1052       // So the file will be sidelined to a similar folder structure.
1053       int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
1054       for (int i = 0; index > 0 && i < 5; i++) {
1055         index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
1056       }
1057       if (index > 0) {
1058         Path rootDir = getSidelineDir();
1059         Path dst = new Path(rootDir, pathStr.substring(index + 1));
1060         fs.mkdirs(dst.getParent());
1061         LOG.info("Trying to sildeline reference file "
1062           + path + " to " + dst);
1063         setShouldRerun();
1064 
1065         success = fs.rename(path, dst);
1066       }
1067       if (!success) {
1068         LOG.error("Failed to sideline reference file " + path);
1069       }
1070     }
1071   }
1072 
1073   /**
1074    * TODO -- need to add tests for this.
1075    */
1076   private void reportEmptyMetaCells() {
1077     errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
1078       emptyRegionInfoQualifiers.size());
1079     if (details) {
1080       for (Result r: emptyRegionInfoQualifiers) {
1081         errors.print("  " + r);
1082       }
1083     }
1084   }
1085 
1086   /**
1087    * TODO -- need to add tests for this.
1088    */
1089   private void reportTablesInFlux() {
1090     AtomicInteger numSkipped = new AtomicInteger(0);
1091     HTableDescriptor[] allTables = getTables(numSkipped);
1092     errors.print("Number of Tables: " + allTables.length);
1093     if (details) {
1094       if (numSkipped.get() > 0) {
1095         errors.detail("Number of Tables in flux: " + numSkipped.get());
1096       }
1097       for (HTableDescriptor td : allTables) {
1098         errors.detail("  Table: " + td.getTableName() + "\t" +
1099                            (td.isReadOnly() ? "ro" : "rw") + "\t" +
1100                             (td.isMetaRegion() ? "META" : "    ") + "\t" +
1101                            " families: " + td.getFamilies().size());
1102       }
1103     }
1104   }
1105 
1106   public ErrorReporter getErrors() {
1107     return errors;
1108   }
1109 
1110   /**
1111    * Read the .regioninfo file from the file system.  If there is no
1112    * .regioninfo, add it to the orphan hdfs region list.
1113    */
1114   private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
1115     Path regionDir = hbi.getHdfsRegionDir();
1116     if (regionDir == null) {
1117       LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
1118       return;
1119     }
1120 
1121     if (hbi.hdfsEntry.hri != null) {
1122       // already loaded data
1123       return;
1124     }
1125 
1126     FileSystem fs = FileSystem.get(getConf());
1127     HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
1128     LOG.debug("HRegionInfo read: " + hri.toString());
1129     hbi.hdfsEntry.hri = hri;
1130   }
1131 
1132   /**
1133    * Exception thrown when a integrity repair operation fails in an
1134    * unresolvable way.
1135    */
1136   public static class RegionRepairException extends IOException {
1137     private static final long serialVersionUID = 1L;
1138     final IOException ioe;
1139     public RegionRepairException(String s, IOException ioe) {
1140       super(s);
1141       this.ioe = ioe;
1142     }
1143   }
1144 
1145   /**
1146    * Populate hbi's from regionInfos loaded from file system.
1147    */
1148   private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
1149       throws IOException, InterruptedException {
1150     tablesInfo.clear(); // regenerating the data
1151     // generate region split structure
1152     Collection<HbckInfo> hbckInfos = regionInfoMap.values();
1153 
1154     // Parallelized read of .regioninfo files.
1155     List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
1156     List<Future<Void>> hbiFutures;
1157 
1158     for (HbckInfo hbi : hbckInfos) {
1159       WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1160       hbis.add(work);
1161     }
1162 
1163     // Submit and wait for completion
1164     hbiFutures = executor.invokeAll(hbis);
1165 
1166     for(int i=0; i<hbiFutures.size(); i++) {
1167       WorkItemHdfsRegionInfo work = hbis.get(i);
1168       Future<Void> f = hbiFutures.get(i);
1169       try {
1170         f.get();
1171       } catch(ExecutionException e) {
1172         LOG.warn("Failed to read .regioninfo file for region " +
1173               work.hbi.getRegionNameAsString(), e.getCause());
1174       }
1175     }
1176 
1177     Path hbaseRoot = FSUtils.getRootDir(getConf());
1178     FileSystem fs = hbaseRoot.getFileSystem(getConf());
1179     // serialized table info gathering.
1180     for (HbckInfo hbi: hbckInfos) {
1181 
1182       if (hbi.getHdfsHRI() == null) {
1183         // was an orphan
1184         continue;
1185       }
1186 
1187 
1188       // get table name from hdfs, populate various HBaseFsck tables.
1189       TableName tableName = hbi.getTableName();
1190       if (tableName == null) {
1191         // There was an entry in hbase:meta not in the HDFS?
1192         LOG.warn("tableName was null for: " + hbi);
1193         continue;
1194       }
1195 
1196       TableInfo modTInfo = tablesInfo.get(tableName);
1197       if (modTInfo == null) {
1198         // only executed once per table.
1199         modTInfo = new TableInfo(tableName);
1200         tablesInfo.put(tableName, modTInfo);
1201         try {
1202           TableDescriptor htd =
1203               FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1204           modTInfo.htds.add(htd.getHTableDescriptor());
1205         } catch (IOException ioe) {
1206           if (!orphanTableDirs.containsKey(tableName)) {
1207             LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1208             //should only report once for each table
1209             errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1210                 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1211             Set<String> columns = new HashSet<String>();
1212             orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1213           }
1214         }
1215       }
1216       if (!hbi.isSkipChecks()) {
1217         modTInfo.addRegionInfo(hbi);
1218       }
1219     }
1220 
1221     loadTableInfosForTablesWithNoRegion();
1222     errors.print("");
1223 
1224     return tablesInfo;
1225   }
1226 
1227   /**
1228    * To get the column family list according to the column family dirs
1229    * @param columns
1230    * @param hbi
1231    * @return a set of column families
1232    * @throws IOException
1233    */
1234   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
1235     Path regionDir = hbi.getHdfsRegionDir();
1236     FileSystem fs = regionDir.getFileSystem(getConf());
1237     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1238     for (FileStatus subdir : subDirs) {
1239       String columnfamily = subdir.getPath().getName();
1240       columns.add(columnfamily);
1241     }
1242     return columns;
1243   }
1244 
1245   /**
1246    * To fabricate a .tableinfo file with following contents<br>
1247    * 1. the correct tablename <br>
1248    * 2. the correct colfamily list<br>
1249    * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1250    * @throws IOException
1251    */
1252   private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1253       Set<String> columns) throws IOException {
1254     if (columns ==null || columns.isEmpty()) return false;
1255     HTableDescriptor htd = new HTableDescriptor(tableName);
1256     for (String columnfamimly : columns) {
1257       htd.addFamily(new HColumnDescriptor(columnfamimly));
1258     }
1259     fstd.createTableDescriptor(new TableDescriptor(htd), true);
1260     return true;
1261   }
1262 
1263   /**
1264    * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1265    * @throws IOException
1266    */
1267   public void fixEmptyMetaCells() throws IOException {
1268     if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1269       LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1270       for (Result region : emptyRegionInfoQualifiers) {
1271         deleteMetaRegion(region.getRow());
1272         errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1273       }
1274       emptyRegionInfoQualifiers.clear();
1275     }
1276   }
1277 
1278   /**
1279    * To fix orphan table by creating a .tableinfo file under tableDir <br>
1280    * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1281    * 2. else create a default .tableinfo file with following items<br>
1282    * &nbsp;2.1 the correct tablename <br>
1283    * &nbsp;2.2 the correct colfamily list<br>
1284    * &nbsp;2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1285    * @throws IOException
1286    */
1287   public void fixOrphanTables() throws IOException {
1288     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1289 
1290       List<TableName> tmpList = new ArrayList<TableName>();
1291       tmpList.addAll(orphanTableDirs.keySet());
1292       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1293       Iterator<Entry<TableName, Set<String>>> iter =
1294           orphanTableDirs.entrySet().iterator();
1295       int j = 0;
1296       int numFailedCase = 0;
1297       FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1298       while (iter.hasNext()) {
1299         Entry<TableName, Set<String>> entry =
1300             iter.next();
1301         TableName tableName = entry.getKey();
1302         LOG.info("Trying to fix orphan table error: " + tableName);
1303         if (j < htds.length) {
1304           if (tableName.equals(htds[j].getTableName())) {
1305             HTableDescriptor htd = htds[j];
1306             LOG.info("fixing orphan table: " + tableName + " from cache");
1307             fstd.createTableDescriptor(new TableDescriptor(htd), true);
1308             j++;
1309             iter.remove();
1310           }
1311         } else {
1312           if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1313             LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1314             LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1315             iter.remove();
1316           } else {
1317             LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1318             numFailedCase++;
1319           }
1320         }
1321         fixes++;
1322       }
1323 
1324       if (orphanTableDirs.isEmpty()) {
1325         // all orphanTableDirs are luckily recovered
1326         // re-run doFsck after recovering the .tableinfo file
1327         setShouldRerun();
1328         LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1329       } else if (numFailedCase > 0) {
1330         LOG.error("Failed to fix " + numFailedCase
1331             + " OrphanTables with default .tableinfo files");
1332       }
1333 
1334     }
1335     //cleanup the list
1336     orphanTableDirs.clear();
1337 
1338   }
1339 
1340   /**
1341    * This borrows code from MasterFileSystem.bootstrap(). Explicitly creates it's own WAL, so be
1342    * sure to close it as well as the region when you're finished.
1343    *
1344    * @return an open hbase:meta HRegion
1345    */
1346   private HRegion createNewMeta() throws IOException {
1347     Path rootdir = FSUtils.getRootDir(getConf());
1348     Configuration c = getConf();
1349     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1350     HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
1351     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false);
1352     // The WAL subsystem will use the default rootDir rather than the passed in rootDir
1353     // unless I pass along via the conf.
1354     Configuration confForWAL = new Configuration(c);
1355     confForWAL.set(HConstants.HBASE_DIR, rootdir.toString());
1356     WAL wal = (new WALFactory(confForWAL,
1357         Collections.<WALActionsListener>singletonList(new MetricsWAL()),
1358         "hbck-meta-recovery-" + RandomStringUtils.randomNumeric(8))).
1359         getWAL(metaHRI.getEncodedNameAsBytes(), metaHRI.getTable().getNamespace());
1360     HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor, wal);
1361     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
1362     return meta;
1363   }
1364 
1365   /**
1366    * Generate set of puts to add to new meta.  This expects the tables to be
1367    * clean with no overlaps or holes.  If there are any problems it returns null.
1368    *
1369    * @return An array list of puts to do in bulk, null if tables have problems
1370    */
1371   private ArrayList<Put> generatePuts(
1372       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1373     ArrayList<Put> puts = new ArrayList<Put>();
1374     boolean hasProblems = false;
1375     for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1376       TableName name = e.getKey();
1377 
1378       // skip "hbase:meta"
1379       if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1380         continue;
1381       }
1382 
1383       TableInfo ti = e.getValue();
1384       puts.add(MetaTableAccessor
1385           .makePutFromTableState(new TableState(ti.tableName, TableState.State.ENABLED)));
1386       for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1387           .entrySet()) {
1388         Collection<HbckInfo> his = spl.getValue();
1389         int sz = his.size();
1390         if (sz != 1) {
1391           // problem
1392           LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1393               + " had " +  sz + " regions instead of exactly 1." );
1394           hasProblems = true;
1395           continue;
1396         }
1397 
1398         // add the row directly to meta.
1399         HbckInfo hi = his.iterator().next();
1400         HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
1401         Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1402         puts.add(p);
1403       }
1404     }
1405     return hasProblems ? null : puts;
1406   }
1407 
1408   /**
1409    * Suggest fixes for each table
1410    */
1411   private void suggestFixes(
1412       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1413     logParallelMerge();
1414     for (TableInfo tInfo : tablesInfo.values()) {
1415       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1416       tInfo.checkRegionChain(handler);
1417     }
1418   }
1419 
1420   /**
1421    * Rebuilds meta from information in hdfs/fs.  Depends on configuration settings passed into
1422    * hbck constructor to point to a particular fs/dir. Assumes HBase is OFFLINE.
1423    *
1424    * @param fix flag that determines if method should attempt to fix holes
1425    * @return true if successful, false if attempt failed.
1426    */
1427   public boolean rebuildMeta(boolean fix) throws IOException,
1428       InterruptedException {
1429 
1430     // TODO check to make sure hbase is offline. (or at least the table
1431     // currently being worked on is off line)
1432 
1433     // Determine what's on HDFS
1434     LOG.info("Loading HBase regioninfo from HDFS...");
1435     loadHdfsRegionDirs(); // populating regioninfo table.
1436 
1437     int errs = errors.getErrorList().size();
1438     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1439     checkHdfsIntegrity(false, false);
1440 
1441     // make sure ok.
1442     if (errors.getErrorList().size() != errs) {
1443       // While in error state, iterate until no more fixes possible
1444       while(true) {
1445         fixes = 0;
1446         suggestFixes(tablesInfo);
1447         errors.clear();
1448         loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1449         checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1450 
1451         int errCount = errors.getErrorList().size();
1452 
1453         if (fixes == 0) {
1454           if (errCount > 0) {
1455             return false; // failed to fix problems.
1456           } else {
1457             break; // no fixes and no problems? drop out and fix stuff!
1458           }
1459         }
1460       }
1461     }
1462 
1463     // we can rebuild, move old meta out of the way and start
1464     LOG.info("HDFS regioninfo's seems good.  Sidelining old hbase:meta");
1465     Path backupDir = sidelineOldMeta();
1466 
1467     LOG.info("Creating new hbase:meta");
1468     HRegion meta = createNewMeta();
1469 
1470     // populate meta
1471     List<Put> puts = generatePuts(tablesInfo);
1472     if (puts == null) {
1473       LOG.fatal("Problem encountered when creating new hbase:meta entries.  " +
1474         "You may need to restore the previously sidelined hbase:meta");
1475       return false;
1476     }
1477     meta.batchMutate(puts.toArray(new Put[puts.size()]), HConstants.NO_NONCE, HConstants.NO_NONCE);
1478     meta.close();
1479     if (meta.getWAL() != null) {
1480       meta.getWAL().close();
1481     }
1482     LOG.info("Success! hbase:meta table rebuilt.");
1483     LOG.info("Old hbase:meta is moved into " + backupDir);
1484     return true;
1485   }
1486 
1487   /**
1488    * Log an appropriate message about whether or not overlapping merges are computed in parallel.
1489    */
1490   private void logParallelMerge() {
1491     if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
1492       LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
1493           " false to run serially.");
1494     } else {
1495       LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
1496           " true to run in parallel.");
1497     }
1498   }
1499 
1500   private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1501       boolean fixOverlaps) throws IOException {
1502     LOG.info("Checking HBase region split map from HDFS data...");
1503     logParallelMerge();
1504     for (TableInfo tInfo : tablesInfo.values()) {
1505       TableIntegrityErrorHandler handler;
1506       if (fixHoles || fixOverlaps) {
1507         handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1508           fixHoles, fixOverlaps);
1509       } else {
1510         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1511       }
1512       if (!tInfo.checkRegionChain(handler)) {
1513         // should dump info as well.
1514         errors.report("Found inconsistency in table " + tInfo.getName());
1515       }
1516     }
1517     return tablesInfo;
1518   }
1519 
1520   private Path getSidelineDir() throws IOException {
1521     if (sidelineDir == null) {
1522       Path hbaseDir = FSUtils.getRootDir(getConf());
1523       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1524       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1525           + startMillis);
1526     }
1527     return sidelineDir;
1528   }
1529 
1530   /**
1531    * Sideline a region dir (instead of deleting it)
1532    */
1533   Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1534     return sidelineRegionDir(fs, null, hi);
1535   }
1536 
1537   /**
1538    * Sideline a region dir (instead of deleting it)
1539    *
1540    * @param parentDir if specified, the region will be sidelined to folder like
1541    *     {@literal .../parentDir/<table name>/<region name>}. The purpose is to group together
1542    *     similar regions sidelined, for example, those regions should be bulk loaded back later
1543    *     on. If NULL, it is ignored.
1544    */
1545   Path sidelineRegionDir(FileSystem fs,
1546       String parentDir, HbckInfo hi) throws IOException {
1547     TableName tableName = hi.getTableName();
1548     Path regionDir = hi.getHdfsRegionDir();
1549 
1550     if (!fs.exists(regionDir)) {
1551       LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1552       return null;
1553     }
1554 
1555     Path rootDir = getSidelineDir();
1556     if (parentDir != null) {
1557       rootDir = new Path(rootDir, parentDir);
1558     }
1559     Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1560     Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1561     fs.mkdirs(sidelineRegionDir);
1562     boolean success = false;
1563     FileStatus[] cfs =  fs.listStatus(regionDir);
1564     if (cfs == null) {
1565       LOG.info("Region dir is empty: " + regionDir);
1566     } else {
1567       for (FileStatus cf : cfs) {
1568         Path src = cf.getPath();
1569         Path dst =  new Path(sidelineRegionDir, src.getName());
1570         if (fs.isFile(src)) {
1571           // simple file
1572           success = fs.rename(src, dst);
1573           if (!success) {
1574             String msg = "Unable to rename file " + src +  " to " + dst;
1575             LOG.error(msg);
1576             throw new IOException(msg);
1577           }
1578           continue;
1579         }
1580 
1581         // is a directory.
1582         fs.mkdirs(dst);
1583 
1584         LOG.info("Sidelining files from " + src + " into containing region " + dst);
1585         // FileSystem.rename is inconsistent with directories -- if the
1586         // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1587         // it moves the src into the dst dir resulting in (foo/a/b).  If
1588         // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1589         FileStatus[] hfiles = fs.listStatus(src);
1590         if (hfiles != null && hfiles.length > 0) {
1591           for (FileStatus hfile : hfiles) {
1592             success = fs.rename(hfile.getPath(), dst);
1593             if (!success) {
1594               String msg = "Unable to rename file " + src +  " to " + dst;
1595               LOG.error(msg);
1596               throw new IOException(msg);
1597             }
1598           }
1599         }
1600         LOG.debug("Sideline directory contents:");
1601         debugLsr(sidelineRegionDir);
1602       }
1603     }
1604 
1605     LOG.info("Removing old region dir: " + regionDir);
1606     success = fs.delete(regionDir, true);
1607     if (!success) {
1608       String msg = "Unable to delete dir " + regionDir;
1609       LOG.error(msg);
1610       throw new IOException(msg);
1611     }
1612     return sidelineRegionDir;
1613   }
1614 
1615   /**
1616    * Side line an entire table.
1617    */
1618   void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1619       Path backupHbaseDir) throws IOException {
1620     Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1621     if (fs.exists(tableDir)) {
1622       Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1623       fs.mkdirs(backupTableDir.getParent());
1624       boolean success = fs.rename(tableDir, backupTableDir);
1625       if (!success) {
1626         throw new IOException("Failed to move  " + tableName + " from "
1627             +  tableDir + " to " + backupTableDir);
1628       }
1629     } else {
1630       LOG.info("No previous " + tableName +  " exists.  Continuing.");
1631     }
1632   }
1633 
1634   /**
1635    * @return Path to backup of original directory
1636    */
1637   Path sidelineOldMeta() throws IOException {
1638     // put current hbase:meta aside.
1639     Path hbaseDir = FSUtils.getRootDir(getConf());
1640     FileSystem fs = hbaseDir.getFileSystem(getConf());
1641     Path backupDir = getSidelineDir();
1642     fs.mkdirs(backupDir);
1643 
1644     try {
1645       sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1646     } catch (IOException e) {
1647         LOG.fatal("... failed to sideline meta. Currently in inconsistent state.  To restore "
1648             + "try to rename hbase:meta in " + backupDir.getName() + " to "
1649             + hbaseDir.getName() + ".", e);
1650       throw e; // throw original exception
1651     }
1652     return backupDir;
1653   }
1654 
1655   /**
1656    * Load the list of disabled tables in ZK into local set.
1657    * @throws ZooKeeperConnectionException
1658    * @throws IOException
1659    */
1660   private void loadTableStates()
1661   throws IOException {
1662     tableStates = MetaTableAccessor.getTableStates(connection);
1663   }
1664 
1665   /**
1666    * Check if the specified region's table is disabled.
1667    * @param tableName table to check status of
1668    */
1669   private boolean isTableDisabled(TableName tableName) {
1670     return tableStates.containsKey(tableName)
1671         && tableStates.get(tableName)
1672         .inStates(TableState.State.DISABLED, TableState.State.DISABLING);
1673   }
1674 
1675   /**
1676    * Scan HDFS for all regions, recording their information into
1677    * regionInfoMap
1678    */
1679   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1680     Path rootDir = FSUtils.getRootDir(getConf());
1681     FileSystem fs = rootDir.getFileSystem(getConf());
1682 
1683     // list all tables from HDFS
1684     List<FileStatus> tableDirs = Lists.newArrayList();
1685 
1686     boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1687 
1688     List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1689     for (Path path : paths) {
1690       TableName tableName = FSUtils.getTableName(path);
1691        if ((!checkMetaOnly &&
1692            isTableIncluded(tableName)) ||
1693            tableName.equals(TableName.META_TABLE_NAME)) {
1694          tableDirs.add(fs.getFileStatus(path));
1695        }
1696     }
1697 
1698     // verify that version file exists
1699     if (!foundVersionFile) {
1700       errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1701           "Version file does not exist in root dir " + rootDir);
1702       if (shouldFixVersionFile()) {
1703         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1704             + " file.");
1705         setShouldRerun();
1706         FSUtils.setVersion(fs, rootDir, getConf().getInt(
1707             HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1708             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1709             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1710       }
1711     }
1712 
1713     // level 1:  <HBASE_DIR>/*
1714     List<WorkItemHdfsDir> dirs = new ArrayList<WorkItemHdfsDir>(tableDirs.size());
1715     List<Future<Void>> dirsFutures;
1716 
1717     for (FileStatus tableDir : tableDirs) {
1718       LOG.debug("Loading region dirs from " +tableDir.getPath());
1719       dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir));
1720     }
1721 
1722     // Invoke and wait for Callables to complete
1723     dirsFutures = executor.invokeAll(dirs);
1724 
1725     for(Future<Void> f: dirsFutures) {
1726       try {
1727         f.get();
1728       } catch(ExecutionException e) {
1729         LOG.warn("Could not load region dir " , e.getCause());
1730       }
1731     }
1732     errors.print("");
1733   }
1734 
1735   /**
1736    * Record the location of the hbase:meta region as found in ZooKeeper.
1737    */
1738   private boolean recordMetaRegion() throws IOException {
1739     RegionLocations rl = ((ClusterConnection)connection).locateRegion(TableName.META_TABLE_NAME,
1740         HConstants.EMPTY_START_ROW, false, false);
1741     if (rl == null) {
1742       errors.reportError(ERROR_CODE.NULL_META_REGION,
1743           "META region was not found in Zookeeper");
1744       return false;
1745     }
1746     for (HRegionLocation metaLocation : rl.getRegionLocations()) {
1747       // Check if Meta region is valid and existing
1748       if (metaLocation == null ) {
1749         errors.reportError(ERROR_CODE.NULL_META_REGION,
1750             "META region location is null");
1751         return false;
1752       }
1753       if (metaLocation.getRegionInfo() == null) {
1754         errors.reportError(ERROR_CODE.NULL_META_REGION,
1755             "META location regionInfo is null");
1756         return false;
1757       }
1758       if (metaLocation.getHostname() == null) {
1759         errors.reportError(ERROR_CODE.NULL_META_REGION,
1760             "META location hostName is null");
1761         return false;
1762       }
1763       ServerName sn = metaLocation.getServerName();
1764       MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, EnvironmentEdgeManager.currentTime());
1765       HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1766       if (hbckInfo == null) {
1767         regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1768       } else {
1769         hbckInfo.metaEntry = m;
1770       }
1771     }
1772     return true;
1773   }
1774 
1775   private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1776     return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1777       @Override
1778       public void abort(String why, Throwable e) {
1779         LOG.error(why, e);
1780         System.exit(1);
1781       }
1782 
1783       @Override
1784       public boolean isAborted() {
1785         return false;
1786       }
1787 
1788     });
1789   }
1790 
1791   private ServerName getMetaRegionServerName(int replicaId)
1792   throws IOException, KeeperException {
1793     return new MetaTableLocator().getMetaRegionLocation(zkw, replicaId);
1794   }
1795 
1796   /**
1797    * Contacts each regionserver and fetches metadata about regions.
1798    * @param regionServerList - the list of region servers to connect to
1799    * @throws IOException if a remote or network exception occurs
1800    */
1801   void processRegionServers(Collection<ServerName> regionServerList)
1802     throws IOException, InterruptedException {
1803 
1804     List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1805     List<Future<Void>> workFutures;
1806 
1807     // loop to contact each region server in parallel
1808     for (ServerName rsinfo: regionServerList) {
1809       workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1810     }
1811 
1812     workFutures = executor.invokeAll(workItems);
1813 
1814     for(int i=0; i<workFutures.size(); i++) {
1815       WorkItemRegion item = workItems.get(i);
1816       Future<Void> f = workFutures.get(i);
1817       try {
1818         f.get();
1819       } catch(ExecutionException e) {
1820         LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1821             e.getCause());
1822       }
1823     }
1824   }
1825 
1826   /**
1827    * Check consistency of all regions that have been found in previous phases.
1828    */
1829   private void checkAndFixConsistency()
1830   throws IOException, KeeperException, InterruptedException {
1831     // Divide the checks in two phases. One for default/primary replicas and another
1832     // for the non-primary ones. Keeps code cleaner this way.
1833 
1834     List<CheckRegionConsistencyWorkItem> workItems =
1835         new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1836     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1837       if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1838         workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1839       }
1840     }
1841     checkRegionConsistencyConcurrently(workItems);
1842 
1843     boolean prevHdfsCheck = shouldCheckHdfs();
1844     setCheckHdfs(false); //replicas don't have any hdfs data
1845     // Run a pass over the replicas and fix any assignment issues that exist on the currently
1846     // deployed/undeployed replicas.
1847     List<CheckRegionConsistencyWorkItem> replicaWorkItems =
1848         new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1849     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1850       if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1851         replicaWorkItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1852       }
1853     }
1854     checkRegionConsistencyConcurrently(replicaWorkItems);
1855     setCheckHdfs(prevHdfsCheck);
1856 
1857     // If some regions is skipped during checkRegionConsistencyConcurrently() phase, we might
1858     // not get accurate state of the hbase if continuing. The config here allows users to tune
1859     // the tolerance of number of skipped region.
1860     // TODO: evaluate the consequence to continue the hbck operation without config.
1861     int terminateThreshold =  getConf().getInt("hbase.hbck.skipped.regions.limit", 0);
1862     int numOfSkippedRegions = skippedRegions.size();
1863     if (numOfSkippedRegions > 0 && numOfSkippedRegions > terminateThreshold) {
1864       throw new IOException(numOfSkippedRegions
1865         + " region(s) could not be checked or repaired.  See logs for detail.");
1866     }
1867 
1868     if (shouldCheckHdfs()) {
1869       checkAndFixTableStates();
1870     }
1871   }
1872 
1873   /**
1874    * Check consistency of all regions using mulitple threads concurrently.
1875    */
1876   private void checkRegionConsistencyConcurrently(
1877     final List<CheckRegionConsistencyWorkItem> workItems)
1878     throws IOException, KeeperException, InterruptedException {
1879     if (workItems.isEmpty()) {
1880       return;  // nothing to check
1881     }
1882 
1883     List<Future<Void>> workFutures = executor.invokeAll(workItems);
1884     for(Future<Void> f: workFutures) {
1885       try {
1886         f.get();
1887       } catch(ExecutionException e1) {
1888         LOG.warn("Could not check region consistency " , e1.getCause());
1889         if (e1.getCause() instanceof IOException) {
1890           throw (IOException)e1.getCause();
1891         } else if (e1.getCause() instanceof KeeperException) {
1892           throw (KeeperException)e1.getCause();
1893         } else if (e1.getCause() instanceof InterruptedException) {
1894           throw (InterruptedException)e1.getCause();
1895         } else {
1896           throw new IOException(e1.getCause());
1897         }
1898       }
1899     }
1900   }
1901 
1902   class CheckRegionConsistencyWorkItem implements Callable<Void> {
1903     private final String key;
1904     private final HbckInfo hbi;
1905 
1906     CheckRegionConsistencyWorkItem(String key, HbckInfo hbi) {
1907       this.key = key;
1908       this.hbi = hbi;
1909     }
1910 
1911     @Override
1912     public synchronized Void call() throws Exception {
1913       try {
1914         checkRegionConsistency(key, hbi);
1915       } catch (Exception e) {
1916         // If the region is non-META region, skip this region and send warning/error message; if
1917         // the region is META region, we should not continue.
1918         LOG.warn("Unable to complete check or repair the region '" + hbi.getRegionNameAsString()
1919           + "'.", e);
1920         if (hbi.getHdfsHRI().isMetaRegion()) {
1921           throw e;
1922         }
1923         LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'");
1924         addSkippedRegion(hbi);
1925       }
1926       return null;
1927     }
1928   }
1929 
1930   private void addSkippedRegion(final HbckInfo hbi) {
1931     Set<String> skippedRegionNames = skippedRegions.get(hbi.getTableName());
1932     if (skippedRegionNames == null) {
1933       skippedRegionNames = new HashSet<String>();
1934     }
1935     skippedRegionNames.add(hbi.getRegionNameAsString());
1936     skippedRegions.put(hbi.getTableName(), skippedRegionNames);
1937   }
1938 
1939   /**
1940    * Check and fix table states, assumes full info available:
1941    * - tableInfos
1942    * - empty tables loaded
1943    */
1944   private void checkAndFixTableStates() throws IOException {
1945     // first check dangling states
1946     for (Entry<TableName, TableState> entry : tableStates.entrySet()) {
1947       TableName tableName = entry.getKey();
1948       TableState tableState = entry.getValue();
1949       TableInfo tableInfo = tablesInfo.get(tableName);
1950       if (isTableIncluded(tableName)
1951           && !tableName.isSystemTable()
1952           && tableInfo == null) {
1953         if (fixMeta) {
1954           MetaTableAccessor.deleteTableState(connection, tableName);
1955           TableState state = MetaTableAccessor.getTableState(connection, tableName);
1956           if (state != null) {
1957             errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,
1958                 tableName + " unable to delete dangling table state " + tableState);
1959           }
1960         } else {
1961           errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,
1962               tableName + " has dangling table state " + tableState);
1963         }
1964       }
1965     }
1966     // check that all tables have states
1967     for (TableName tableName : tablesInfo.keySet()) {
1968       if (isTableIncluded(tableName) && !tableStates.containsKey(tableName)) {
1969         if (fixMeta) {
1970           MetaTableAccessor.updateTableState(connection, tableName, TableState.State.ENABLED);
1971           TableState newState = MetaTableAccessor.getTableState(connection, tableName);
1972           if (newState == null) {
1973             errors.reportError(ERROR_CODE.NO_TABLE_STATE,
1974                 "Unable to change state for table " + tableName + " in meta ");
1975           }
1976         } else {
1977           errors.reportError(ERROR_CODE.NO_TABLE_STATE,
1978               tableName + " has no state in meta ");
1979         }
1980       }
1981     }
1982   }
1983 
1984   private void preCheckPermission() throws IOException, AccessDeniedException {
1985     if (shouldIgnorePreCheckPermission()) {
1986       return;
1987     }
1988 
1989     Path hbaseDir = FSUtils.getRootDir(getConf());
1990     FileSystem fs = hbaseDir.getFileSystem(getConf());
1991     UserProvider userProvider = UserProvider.instantiate(getConf());
1992     UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1993     FileStatus[] files = fs.listStatus(hbaseDir);
1994     for (FileStatus file : files) {
1995       try {
1996         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1997       } catch (AccessDeniedException ace) {
1998         LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1999         errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
2000           + " does not have write perms to " + file.getPath()
2001           + ". Please rerun hbck as hdfs user " + file.getOwner());
2002         throw ace;
2003       }
2004     }
2005   }
2006 
2007   /**
2008    * Deletes region from meta table
2009    */
2010   private void deleteMetaRegion(HbckInfo hi) throws IOException {
2011     deleteMetaRegion(hi.metaEntry.getRegionName());
2012   }
2013 
2014   /**
2015    * Deletes region from meta table
2016    */
2017   private void deleteMetaRegion(byte[] metaKey) throws IOException {
2018     Delete d = new Delete(metaKey);
2019     meta.delete(d);
2020     LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
2021   }
2022 
2023   /**
2024    * Reset the split parent region info in meta table
2025    */
2026   private void resetSplitParent(HbckInfo hi) throws IOException {
2027     RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
2028     Delete d = new Delete(hi.metaEntry.getRegionName());
2029     d.addColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
2030     d.addColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
2031     mutations.add(d);
2032 
2033     HRegionInfo hri = new HRegionInfo(hi.metaEntry);
2034     hri.setOffline(false);
2035     hri.setSplit(false);
2036     Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
2037     mutations.add(p);
2038 
2039     meta.mutateRow(mutations);
2040     LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
2041   }
2042 
2043   /**
2044    * This backwards-compatibility wrapper for permanently offlining a region
2045    * that should not be alive.  If the region server does not support the
2046    * "offline" method, it will use the closest unassign method instead.  This
2047    * will basically work until one attempts to disable or delete the affected
2048    * table.  The problem has to do with in-memory only master state, so
2049    * restarting the HMaster or failing over to another should fix this.
2050    */
2051   private void offline(byte[] regionName) throws IOException {
2052     String regionString = Bytes.toStringBinary(regionName);
2053     if (!rsSupportsOffline) {
2054       LOG.warn("Using unassign region " + regionString
2055           + " instead of using offline method, you should"
2056           + " restart HMaster after these repairs");
2057       admin.unassign(regionName, true);
2058       return;
2059     }
2060 
2061     // first time we assume the rs's supports #offline.
2062     try {
2063       LOG.info("Offlining region " + regionString);
2064       admin.offline(regionName);
2065     } catch (IOException ioe) {
2066       String notFoundMsg = "java.lang.NoSuchMethodException: " +
2067         "org.apache.hadoop.hbase.master.HMaster.offline([B)";
2068       if (ioe.getMessage().contains(notFoundMsg)) {
2069         LOG.warn("Using unassign region " + regionString
2070             + " instead of using offline method, you should"
2071             + " restart HMaster after these repairs");
2072         rsSupportsOffline = false; // in the future just use unassign
2073         admin.unassign(regionName, true);
2074         return;
2075       }
2076       throw ioe;
2077     }
2078   }
2079 
2080   private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
2081     undeployRegionsForHbi(hi);
2082     // undeploy replicas of the region (but only if the method is invoked for the primary)
2083     if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2084       return;
2085     }
2086     int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2087     for (int i = 1; i < numReplicas; i++) {
2088       if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
2089       HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
2090           hi.getPrimaryHRIForDeployedReplica(), i);
2091       HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2092       if (h != null) {
2093         undeployRegionsForHbi(h);
2094         //set skip checks; we undeployed it, and we don't want to evaluate this anymore
2095         //in consistency checks
2096         h.setSkipChecks(true);
2097       }
2098     }
2099   }
2100 
2101   private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
2102     for (OnlineEntry rse : hi.deployedEntries) {
2103       LOG.debug("Undeploy region "  + rse.hri + " from " + rse.hsa);
2104       try {
2105         HBaseFsckRepair.closeRegionSilentlyAndWait(connection, rse.hsa, rse.hri);
2106         offline(rse.hri.getRegionName());
2107       } catch (IOException ioe) {
2108         LOG.warn("Got exception when attempting to offline region "
2109             + Bytes.toString(rse.hri.getRegionName()), ioe);
2110       }
2111     }
2112   }
2113 
2114   /**
2115    * Attempts to undeploy a region from a region server based in information in
2116    * META.  Any operations that modify the file system should make sure that
2117    * its corresponding region is not deployed to prevent data races.
2118    *
2119    * A separate call is required to update the master in-memory region state
2120    * kept in the AssignementManager.  Because disable uses this state instead of
2121    * that found in META, we can't seem to cleanly disable/delete tables that
2122    * have been hbck fixed.  When used on a version of HBase that does not have
2123    * the offline ipc call exposed on the master (&lt;0.90.5, &lt;0.92.0) a master
2124    * restart or failover may be required.
2125    */
2126   private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
2127     if (hi.metaEntry == null && hi.hdfsEntry == null) {
2128       undeployRegions(hi);
2129       return;
2130     }
2131 
2132     // get assignment info and hregioninfo from meta.
2133     Get get = new Get(hi.getRegionName());
2134     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2135     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
2136     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
2137     // also get the locations of the replicas to close if the primary region is being closed
2138     if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2139       int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2140       for (int i = 0; i < numReplicas; i++) {
2141         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
2142         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
2143       }
2144     }
2145     Result r = meta.get(get);
2146     RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
2147     if (rl == null) {
2148       LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
2149           " since meta does not have handle to reach it");
2150       return;
2151     }
2152     for (HRegionLocation h : rl.getRegionLocations()) {
2153       ServerName serverName = h.getServerName();
2154       if (serverName == null) {
2155         errors.reportError("Unable to close region "
2156             + hi.getRegionNameAsString() +  " because meta does not "
2157             + "have handle to reach it.");
2158         continue;
2159       }
2160       HRegionInfo hri = h.getRegionInfo();
2161       if (hri == null) {
2162         LOG.warn("Unable to close region " + hi.getRegionNameAsString()
2163             + " because hbase:meta had invalid or missing "
2164             + HConstants.CATALOG_FAMILY_STR + ":"
2165             + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
2166             + " qualifier value.");
2167         continue;
2168       }
2169       // close the region -- close files and remove assignment
2170       HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
2171     }
2172   }
2173 
2174   private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
2175     KeeperException, InterruptedException {
2176     // If we are trying to fix the errors
2177     if (shouldFixAssignments()) {
2178       errors.print(msg);
2179       undeployRegions(hbi);
2180       setShouldRerun();
2181       HRegionInfo hri = hbi.getHdfsHRI();
2182       if (hri == null) {
2183         hri = hbi.metaEntry;
2184       }
2185       HBaseFsckRepair.fixUnassigned(admin, hri);
2186       HBaseFsckRepair.waitUntilAssigned(admin, hri);
2187 
2188       // also assign replicas if needed (do it only when this call operates on a primary replica)
2189       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
2190       int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
2191       for (int i = 1; i < replicationCount; i++) {
2192         hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
2193         HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2194         if (h != null) {
2195           undeployRegions(h);
2196           //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore
2197           //in consistency checks
2198           h.setSkipChecks(true);
2199         }
2200         HBaseFsckRepair.fixUnassigned(admin, hri);
2201         HBaseFsckRepair.waitUntilAssigned(admin, hri);
2202       }
2203 
2204     }
2205   }
2206 
2207   /**
2208    * Check a single region for consistency and correct deployment.
2209    */
2210   private void checkRegionConsistency(final String key, final HbckInfo hbi)
2211   throws IOException, KeeperException, InterruptedException {
2212 
2213     if (hbi.isSkipChecks()) return;
2214     String descriptiveName = hbi.toString();
2215     boolean inMeta = hbi.metaEntry != null;
2216     // In case not checking HDFS, assume the region is on HDFS
2217     boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
2218     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
2219     boolean isDeployed = !hbi.deployedOn.isEmpty();
2220     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
2221     boolean deploymentMatchesMeta =
2222       hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
2223       hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
2224     boolean splitParent =
2225         inMeta && hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
2226     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry.getTable());
2227     boolean recentlyModified = inHdfs &&
2228       hbi.getModTime() + timelag > EnvironmentEdgeManager.currentTime();
2229 
2230     // ========== First the healthy cases =============
2231     if (hbi.containsOnlyHdfsEdits()) {
2232       return;
2233     }
2234     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
2235       return;
2236     } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
2237       LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
2238         "tabled that is not deployed");
2239       return;
2240     } else if (recentlyModified) {
2241       LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
2242       return;
2243     }
2244     // ========== Cases where the region is not in hbase:meta =============
2245     else if (!inMeta && !inHdfs && !isDeployed) {
2246       // We shouldn't have record of this region at all then!
2247       assert false : "Entry for region with no data";
2248     } else if (!inMeta && !inHdfs && isDeployed) {
2249       errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
2250           + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
2251           "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2252       if (shouldFixAssignments()) {
2253         undeployRegions(hbi);
2254       }
2255 
2256     } else if (!inMeta && inHdfs && !isDeployed) {
2257       if (hbi.isMerged()) {
2258         // This region has already been merged, the remaining hdfs file will be
2259         // cleaned by CatalogJanitor later
2260         hbi.setSkipChecks(true);
2261         LOG.info("Region " + descriptiveName
2262             + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
2263         return;
2264       }
2265       errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
2266           + descriptiveName + " on HDFS, but not listed in hbase:meta " +
2267           "or deployed on any region server");
2268       // restore region consistency of an adopted orphan
2269       if (shouldFixMeta()) {
2270         if (!hbi.isHdfsRegioninfoPresent()) {
2271           LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
2272               +  " in table integrity repair phase if -fixHdfsOrphans was" +
2273               " used.");
2274           return;
2275         }
2276 
2277         HRegionInfo hri = hbi.getHdfsHRI();
2278         TableInfo tableInfo = tablesInfo.get(hri.getTable());
2279 
2280         for (HRegionInfo region : tableInfo.getRegionsFromMeta()) {
2281           if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0
2282               && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(),
2283                 hri.getEndKey()) >= 0)
2284               && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) {
2285             if(region.isSplit() || region.isOffline()) continue;
2286             Path regionDir = hbi.getHdfsRegionDir();
2287             FileSystem fs = regionDir.getFileSystem(getConf());
2288             List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir);
2289             for (Path familyDir : familyDirs) {
2290               List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);
2291               for (Path referenceFilePath : referenceFilePaths) {
2292                 Path parentRegionDir =
2293                     StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();
2294                 if (parentRegionDir.toString().endsWith(region.getEncodedName())) {
2295                   LOG.warn(hri + " start and stop keys are in the range of " + region
2296                       + ". The region might not be cleaned up from hdfs when region " + region
2297                       + " split failed. Hence deleting from hdfs.");
2298                   HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
2299                     regionDir.getParent(), hri);
2300                   return;
2301                 }
2302               }
2303             }
2304           }
2305         }
2306         LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
2307         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2308         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2309             admin.getClusterStatus().getServers(), numReplicas);
2310 
2311         tryAssignmentRepair(hbi, "Trying to reassign region...");
2312       }
2313 
2314     } else if (!inMeta && inHdfs && isDeployed) {
2315       errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
2316           + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2317       debugLsr(hbi.getHdfsRegionDir());
2318       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2319         // for replicas, this means that we should undeploy the region (we would have
2320         // gone over the primaries and fixed meta holes in first phase under
2321         // checkAndFixConsistency; we shouldn't get the condition !inMeta at
2322         // this stage unless unwanted replica)
2323         if (shouldFixAssignments()) {
2324           undeployRegionsForHbi(hbi);
2325         }
2326       }
2327       if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2328         if (!hbi.isHdfsRegioninfoPresent()) {
2329           LOG.error("This should have been repaired in table integrity repair phase");
2330           return;
2331         }
2332 
2333         LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
2334         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2335         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2336             admin.getClusterStatus().getServers(), numReplicas);
2337         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2338       }
2339 
2340     // ========== Cases where the region is in hbase:meta =============
2341     } else if (inMeta && inHdfs && !isDeployed && splitParent) {
2342       // check whether this is an actual error, or just transient state where parent
2343       // is not cleaned
2344       if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
2345         // check that split daughters are there
2346         HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
2347         HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
2348         if (infoA != null && infoB != null) {
2349           // we already processed or will process daughters. Move on, nothing to see here.
2350           hbi.setSkipChecks(true);
2351           return;
2352         }
2353       }
2354       errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2355           + descriptiveName + " is a split parent in META, in HDFS, "
2356           + "and not deployed on any region server. This could be transient.");
2357       if (shouldFixSplitParents()) {
2358         setShouldRerun();
2359         resetSplitParent(hbi);
2360       }
2361     } else if (inMeta && !inHdfs && !isDeployed) {
2362       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2363           + descriptiveName + " found in META, but not in HDFS "
2364           + "or deployed on any region server.");
2365       if (shouldFixMeta()) {
2366         deleteMetaRegion(hbi);
2367       }
2368     } else if (inMeta && !inHdfs && isDeployed) {
2369       errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2370           + " found in META, but not in HDFS, " +
2371           "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2372       // We treat HDFS as ground truth.  Any information in meta is transient
2373       // and equivalent data can be regenerated.  So, lets unassign and remove
2374       // these problems from META.
2375       if (shouldFixAssignments()) {
2376         errors.print("Trying to fix unassigned region...");
2377         undeployRegions(hbi);
2378       }
2379       if (shouldFixMeta()) {
2380         // wait for it to complete
2381         deleteMetaRegion(hbi);
2382       }
2383     } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2384       errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2385           + " not deployed on any region server.");
2386       tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2387     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2388       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2389           "Region " + descriptiveName + " should not be deployed according " +
2390           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2391       if (shouldFixAssignments()) {
2392         errors.print("Trying to close the region " + descriptiveName);
2393         setShouldRerun();
2394         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2395       }
2396     } else if (inMeta && inHdfs && isMultiplyDeployed) {
2397       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2398           + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
2399           + " but is multiply assigned to region servers " +
2400           Joiner.on(", ").join(hbi.deployedOn));
2401       // If we are trying to fix the errors
2402       if (shouldFixAssignments()) {
2403         errors.print("Trying to fix assignment error...");
2404         setShouldRerun();
2405         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2406       }
2407     } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2408       errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2409           + descriptiveName + " listed in hbase:meta on region server " +
2410           hbi.metaEntry.regionServer + " but found on region server " +
2411           hbi.deployedOn.get(0));
2412       // If we are trying to fix the errors
2413       if (shouldFixAssignments()) {
2414         errors.print("Trying to fix assignment error...");
2415         setShouldRerun();
2416         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2417         HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2418       }
2419     } else {
2420       errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2421           " is in an unforeseen state:" +
2422           " inMeta=" + inMeta +
2423           " inHdfs=" + inHdfs +
2424           " isDeployed=" + isDeployed +
2425           " isMultiplyDeployed=" + isMultiplyDeployed +
2426           " deploymentMatchesMeta=" + deploymentMatchesMeta +
2427           " shouldBeDeployed=" + shouldBeDeployed);
2428     }
2429   }
2430 
2431   /**
2432    * Checks tables integrity. Goes over all regions and scans the tables.
2433    * Collects all the pieces for each table and checks if there are missing,
2434    * repeated or overlapping ones.
2435    * @throws IOException
2436    */
2437   SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2438     tablesInfo = new TreeMap<TableName,TableInfo> ();
2439     LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2440     for (HbckInfo hbi : regionInfoMap.values()) {
2441       // Check only valid, working regions
2442       if (hbi.metaEntry == null) {
2443         // this assumes that consistency check has run loadMetaEntry
2444         Path p = hbi.getHdfsRegionDir();
2445         if (p == null) {
2446           errors.report("No regioninfo in Meta or HDFS. " + hbi);
2447         }
2448 
2449         // TODO test.
2450         continue;
2451       }
2452       if (hbi.metaEntry.regionServer == null) {
2453         errors.detail("Skipping region because no region server: " + hbi);
2454         continue;
2455       }
2456       if (hbi.metaEntry.isOffline()) {
2457         errors.detail("Skipping region because it is offline: " + hbi);
2458         continue;
2459       }
2460       if (hbi.containsOnlyHdfsEdits()) {
2461         errors.detail("Skipping region because it only contains edits" + hbi);
2462         continue;
2463       }
2464 
2465       // Missing regionDir or over-deployment is checked elsewhere. Include
2466       // these cases in modTInfo, so we can evaluate those regions as part of
2467       // the region chain in META
2468       //if (hbi.foundRegionDir == null) continue;
2469       //if (hbi.deployedOn.size() != 1) continue;
2470       if (hbi.deployedOn.size() == 0) continue;
2471 
2472       // We should be safe here
2473       TableName tableName = hbi.metaEntry.getTable();
2474       TableInfo modTInfo = tablesInfo.get(tableName);
2475       if (modTInfo == null) {
2476         modTInfo = new TableInfo(tableName);
2477       }
2478       for (ServerName server : hbi.deployedOn) {
2479         modTInfo.addServer(server);
2480       }
2481 
2482       if (!hbi.isSkipChecks()) {
2483         modTInfo.addRegionInfo(hbi);
2484       }
2485 
2486       tablesInfo.put(tableName, modTInfo);
2487     }
2488 
2489     loadTableInfosForTablesWithNoRegion();
2490 
2491     logParallelMerge();
2492     for (TableInfo tInfo : tablesInfo.values()) {
2493       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2494       if (!tInfo.checkRegionChain(handler)) {
2495         errors.report("Found inconsistency in table " + tInfo.getName());
2496       }
2497     }
2498     return tablesInfo;
2499   }
2500 
2501   /** Loads table info's for tables that may not have been included, since there are no
2502    * regions reported for the table, but table dir is there in hdfs
2503    */
2504   private void loadTableInfosForTablesWithNoRegion() throws IOException {
2505     Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2506     for (HTableDescriptor htd : allTables.values()) {
2507       if (checkMetaOnly && !htd.isMetaTable()) {
2508         continue;
2509       }
2510 
2511       TableName tableName = htd.getTableName();
2512       if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2513         TableInfo tableInfo = new TableInfo(tableName);
2514         tableInfo.htds.add(htd);
2515         tablesInfo.put(htd.getTableName(), tableInfo);
2516       }
2517     }
2518   }
2519 
2520   /**
2521    * Merge hdfs data by moving from contained HbckInfo into targetRegionDir.
2522    * @return number of file move fixes done to merge regions.
2523    */
2524   public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2525     int fileMoves = 0;
2526     String thread = Thread.currentThread().getName();
2527     LOG.debug("[" + thread + "] Contained region dir after close and pause");
2528     debugLsr(contained.getHdfsRegionDir());
2529 
2530     // rename the contained into the container.
2531     FileSystem fs = targetRegionDir.getFileSystem(getConf());
2532     FileStatus[] dirs = null;
2533     try {
2534       dirs = fs.listStatus(contained.getHdfsRegionDir());
2535     } catch (FileNotFoundException fnfe) {
2536       // region we are attempting to merge in is not present!  Since this is a merge, there is
2537       // no harm skipping this region if it does not exist.
2538       if (!fs.exists(contained.getHdfsRegionDir())) {
2539         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2540             + " is missing. Assuming already sidelined or moved.");
2541       } else {
2542         sidelineRegionDir(fs, contained);
2543       }
2544       return fileMoves;
2545     }
2546 
2547     if (dirs == null) {
2548       if (!fs.exists(contained.getHdfsRegionDir())) {
2549         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2550             + " already sidelined.");
2551       } else {
2552         sidelineRegionDir(fs, contained);
2553       }
2554       return fileMoves;
2555     }
2556 
2557     for (FileStatus cf : dirs) {
2558       Path src = cf.getPath();
2559       Path dst =  new Path(targetRegionDir, src.getName());
2560 
2561       if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2562         // do not copy the old .regioninfo file.
2563         continue;
2564       }
2565 
2566       if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2567         // do not copy the .oldlogs files
2568         continue;
2569       }
2570 
2571       LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2572       // FileSystem.rename is inconsistent with directories -- if the
2573       // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2574       // it moves the src into the dst dir resulting in (foo/a/b).  If
2575       // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2576       for (FileStatus hfile : fs.listStatus(src)) {
2577         boolean success = fs.rename(hfile.getPath(), dst);
2578         if (success) {
2579           fileMoves++;
2580         }
2581       }
2582       LOG.debug("[" + thread + "] Sideline directory contents:");
2583       debugLsr(targetRegionDir);
2584     }
2585 
2586     // if all success.
2587     sidelineRegionDir(fs, contained);
2588     LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2589         getSidelineDir());
2590     debugLsr(contained.getHdfsRegionDir());
2591 
2592     return fileMoves;
2593   }
2594 
2595 
2596   static class WorkItemOverlapMerge implements Callable<Void> {
2597     private TableIntegrityErrorHandler handler;
2598     Collection<HbckInfo> overlapgroup;
2599 
2600     WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2601       this.handler = handler;
2602       this.overlapgroup = overlapgroup;
2603     }
2604 
2605     @Override
2606     public Void call() throws Exception {
2607       handler.handleOverlapGroup(overlapgroup);
2608       return null;
2609     }
2610   };
2611 
2612 
2613   /**
2614    * Maintain information about a particular table.
2615    */
2616   public class TableInfo {
2617     TableName tableName;
2618     TreeSet <ServerName> deployedOn;
2619 
2620     // backwards regions
2621     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2622 
2623     // sidelined big overlapped regions
2624     final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2625 
2626     // region split calculator
2627     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2628 
2629     // Histogram of different HTableDescriptors found.  Ideally there is only one!
2630     final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2631 
2632     // key = start split, values = set of splits in problem group
2633     final Multimap<byte[], HbckInfo> overlapGroups =
2634       TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2635 
2636     // list of regions derived from meta entries.
2637     private ImmutableList<HRegionInfo> regionsFromMeta = null;
2638 
2639     TableInfo(TableName name) {
2640       this.tableName = name;
2641       deployedOn = new TreeSet <ServerName>();
2642     }
2643 
2644     /**
2645      * @return descriptor common to all regions.  null if are none or multiple!
2646      */
2647     private HTableDescriptor getHTD() {
2648       if (htds.size() == 1) {
2649         return (HTableDescriptor)htds.toArray()[0];
2650       } else {
2651         LOG.error("None/Multiple table descriptors found for table '"
2652           + tableName + "' regions: " + htds);
2653       }
2654       return null;
2655     }
2656 
2657     public void addRegionInfo(HbckInfo hir) {
2658       if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2659         // end key is absolute end key, just add it.
2660         // ignore replicas other than primary for these checks
2661         if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2662         return;
2663       }
2664 
2665       // if not the absolute end key, check for cycle
2666       if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2667         errors.reportError(
2668             ERROR_CODE.REGION_CYCLE,
2669             String.format("The endkey for this region comes before the "
2670                 + "startkey, startkey=%s, endkey=%s",
2671                 Bytes.toStringBinary(hir.getStartKey()),
2672                 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2673         backwards.add(hir);
2674         return;
2675       }
2676 
2677       // main case, add to split calculator
2678       // ignore replicas other than primary for these checks
2679       if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2680     }
2681 
2682     public void addServer(ServerName server) {
2683       this.deployedOn.add(server);
2684     }
2685 
2686     public TableName getName() {
2687       return tableName;
2688     }
2689 
2690     public int getNumRegions() {
2691       return sc.getStarts().size() + backwards.size();
2692     }
2693 
2694     public synchronized ImmutableList<HRegionInfo> getRegionsFromMeta() {
2695       // lazy loaded, synchronized to ensure a single load
2696       if (regionsFromMeta == null) {
2697         List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
2698         for (HbckInfo h : HBaseFsck.this.regionInfoMap.values()) {
2699           if (tableName.equals(h.getTableName())) {
2700             if (h.metaEntry != null) {
2701               regions.add((HRegionInfo) h.metaEntry);
2702             }
2703           }
2704         }
2705         regionsFromMeta = Ordering.natural().immutableSortedCopy(regions);
2706       }
2707 
2708       return regionsFromMeta;
2709     }
2710 
2711     private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2712       ErrorReporter errors;
2713 
2714       IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2715         this.errors = errors;
2716         setTableInfo(ti);
2717       }
2718 
2719       @Override
2720       public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2721         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2722             "First region should start with an empty key.  You need to "
2723             + " create a new region and regioninfo in HDFS to plug the hole.",
2724             getTableInfo(), hi);
2725       }
2726 
2727       @Override
2728       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2729         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2730             "Last region should end with an empty key. You need to "
2731                 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2732       }
2733 
2734       @Override
2735       public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2736         errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2737             "Region has the same start and end key.", getTableInfo(), hi);
2738       }
2739 
2740       @Override
2741       public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2742         byte[] key = r1.getStartKey();
2743         // dup start key
2744         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2745             "Multiple regions have the same startkey: "
2746             + Bytes.toStringBinary(key), getTableInfo(), r1);
2747         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2748             "Multiple regions have the same startkey: "
2749             + Bytes.toStringBinary(key), getTableInfo(), r2);
2750       }
2751 
2752       @Override
2753       public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2754         errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2755             "There is an overlap in the region chain.",
2756             getTableInfo(), hi1, hi2);
2757       }
2758 
2759       @Override
2760       public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2761         errors.reportError(
2762             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2763             "There is a hole in the region chain between "
2764                 + Bytes.toStringBinary(holeStart) + " and "
2765                 + Bytes.toStringBinary(holeStop)
2766                 + ".  You need to create a new .regioninfo and region "
2767                 + "dir in hdfs to plug the hole.");
2768       }
2769     };
2770 
2771     /**
2772      * This handler fixes integrity errors from hdfs information.  There are
2773      * basically three classes of integrity problems 1) holes, 2) overlaps, and
2774      * 3) invalid regions.
2775      *
2776      * This class overrides methods that fix holes and the overlap group case.
2777      * Individual cases of particular overlaps are handled by the general
2778      * overlap group merge repair case.
2779      *
2780      * If hbase is online, this forces regions offline before doing merge
2781      * operations.
2782      */
2783     private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2784       Configuration conf;
2785 
2786       boolean fixOverlaps = true;
2787 
2788       HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2789           boolean fixHoles, boolean fixOverlaps) {
2790         super(ti, errors);
2791         this.conf = conf;
2792         this.fixOverlaps = fixOverlaps;
2793         // TODO properly use fixHoles
2794       }
2795 
2796       /**
2797        * This is a special case hole -- when the first region of a table is
2798        * missing from META, HBase doesn't acknowledge the existance of the
2799        * table.
2800        */
2801       @Override
2802       public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2803         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2804             "First region should start with an empty key.  Creating a new " +
2805             "region and regioninfo in HDFS to plug the hole.",
2806             getTableInfo(), next);
2807         HTableDescriptor htd = getTableInfo().getHTD();
2808         // from special EMPTY_START_ROW to next region's startKey
2809         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2810             HConstants.EMPTY_START_ROW, next.getStartKey());
2811 
2812         // TODO test
2813         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2814         LOG.info("Table region start key was not empty.  Created new empty region: "
2815             + newRegion + " " +region);
2816         fixes++;
2817       }
2818 
2819       @Override
2820       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2821         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2822             "Last region should end with an empty key.  Creating a new "
2823                 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2824         HTableDescriptor htd = getTableInfo().getHTD();
2825         // from curEndKey to EMPTY_START_ROW
2826         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2827             HConstants.EMPTY_START_ROW);
2828 
2829         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2830         LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
2831             + " " + region);
2832         fixes++;
2833       }
2834 
2835       /**
2836        * There is a hole in the hdfs regions that violates the table integrity
2837        * rules.  Create a new empty region that patches the hole.
2838        */
2839       @Override
2840       public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2841         errors.reportError(
2842             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2843             "There is a hole in the region chain between "
2844                 + Bytes.toStringBinary(holeStartKey) + " and "
2845                 + Bytes.toStringBinary(holeStopKey)
2846                 + ".  Creating a new regioninfo and region "
2847                 + "dir in hdfs to plug the hole.");
2848         HTableDescriptor htd = getTableInfo().getHTD();
2849         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2850         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2851         LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2852         fixes++;
2853       }
2854 
2855       /**
2856        * This takes set of overlapping regions and merges them into a single
2857        * region.  This covers cases like degenerate regions, shared start key,
2858        * general overlaps, duplicate ranges, and partial overlapping regions.
2859        *
2860        * Cases:
2861        * - Clean regions that overlap
2862        * - Only .oldlogs regions (can't find start/stop range, or figure out)
2863        *
2864        * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
2865        */
2866       @Override
2867       public void handleOverlapGroup(Collection<HbckInfo> overlap)
2868           throws IOException {
2869         Preconditions.checkNotNull(overlap);
2870         Preconditions.checkArgument(overlap.size() >0);
2871 
2872         if (!this.fixOverlaps) {
2873           LOG.warn("Not attempting to repair overlaps.");
2874           return;
2875         }
2876 
2877         if (overlap.size() > maxMerge) {
2878           LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2879             "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2880           if (sidelineBigOverlaps) {
2881             // we only sideline big overlapped groups that exceeds the max number of regions to merge
2882             sidelineBigOverlaps(overlap);
2883           }
2884           return;
2885         }
2886 
2887         mergeOverlaps(overlap);
2888       }
2889 
2890       void mergeOverlaps(Collection<HbckInfo> overlap)
2891           throws IOException {
2892         String thread = Thread.currentThread().getName();
2893         LOG.info("== [" + thread + "] Merging regions into one region: "
2894           + Joiner.on(",").join(overlap));
2895         // get the min / max range and close all concerned regions
2896         Pair<byte[], byte[]> range = null;
2897         for (HbckInfo hi : overlap) {
2898           if (range == null) {
2899             range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2900           } else {
2901             if (RegionSplitCalculator.BYTES_COMPARATOR
2902                 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2903               range.setFirst(hi.getStartKey());
2904             }
2905             if (RegionSplitCalculator.BYTES_COMPARATOR
2906                 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2907               range.setSecond(hi.getEndKey());
2908             }
2909           }
2910           // need to close files so delete can happen.
2911           LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
2912           LOG.debug("[" + thread + "] Contained region dir before close");
2913           debugLsr(hi.getHdfsRegionDir());
2914           try {
2915             LOG.info("[" + thread + "] Closing region: " + hi);
2916             closeRegion(hi);
2917           } catch (IOException ioe) {
2918             LOG.warn("[" + thread + "] Was unable to close region " + hi
2919               + ".  Just continuing... ", ioe);
2920           } catch (InterruptedException e) {
2921             LOG.warn("[" + thread + "] Was unable to close region " + hi
2922               + ".  Just continuing... ", e);
2923           }
2924 
2925           try {
2926             LOG.info("[" + thread + "] Offlining region: " + hi);
2927             offline(hi.getRegionName());
2928           } catch (IOException ioe) {
2929             LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2930               + ".  Just continuing... ", ioe);
2931           }
2932         }
2933 
2934         // create new empty container region.
2935         HTableDescriptor htd = getTableInfo().getHTD();
2936         // from start key to end Key
2937         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
2938             range.getSecond());
2939         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2940         LOG.info("[" + thread + "] Created new empty container region: " +
2941             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
2942         debugLsr(region.getRegionFileSystem().getRegionDir());
2943 
2944         // all target regions are closed, should be able to safely cleanup.
2945         boolean didFix= false;
2946         Path target = region.getRegionFileSystem().getRegionDir();
2947         for (HbckInfo contained : overlap) {
2948           LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
2949           int merges = mergeRegionDirs(target, contained);
2950           if (merges > 0) {
2951             didFix = true;
2952           }
2953         }
2954         if (didFix) {
2955           fixes++;
2956         }
2957       }
2958 
2959       /**
2960        * Sideline some regions in a big overlap group so that it
2961        * will have fewer regions, and it is easier to merge them later on.
2962        *
2963        * @param bigOverlap the overlapped group with regions more than maxMerge
2964        * @throws IOException
2965        */
2966       void sidelineBigOverlaps(
2967           Collection<HbckInfo> bigOverlap) throws IOException {
2968         int overlapsToSideline = bigOverlap.size() - maxMerge;
2969         if (overlapsToSideline > maxOverlapsToSideline) {
2970           overlapsToSideline = maxOverlapsToSideline;
2971         }
2972         List<HbckInfo> regionsToSideline =
2973           RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
2974         FileSystem fs = FileSystem.get(conf);
2975         for (HbckInfo regionToSideline: regionsToSideline) {
2976           try {
2977             LOG.info("Closing region: " + regionToSideline);
2978             closeRegion(regionToSideline);
2979           } catch (IOException ioe) {
2980             LOG.warn("Was unable to close region " + regionToSideline
2981               + ".  Just continuing... ", ioe);
2982           } catch (InterruptedException e) {
2983             LOG.warn("Was unable to close region " + regionToSideline
2984               + ".  Just continuing... ", e);
2985           }
2986 
2987           try {
2988             LOG.info("Offlining region: " + regionToSideline);
2989             offline(regionToSideline.getRegionName());
2990           } catch (IOException ioe) {
2991             LOG.warn("Unable to offline region from master: " + regionToSideline
2992               + ".  Just continuing... ", ioe);
2993           }
2994 
2995           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
2996           Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
2997           if (sidelineRegionDir != null) {
2998             sidelinedRegions.put(sidelineRegionDir, regionToSideline);
2999             LOG.info("After sidelined big overlapped region: "
3000               + regionToSideline.getRegionNameAsString()
3001               + " to " + sidelineRegionDir.toString());
3002             fixes++;
3003           }
3004         }
3005       }
3006     }
3007 
3008     /**
3009      * Check the region chain (from META) of this table.  We are looking for
3010      * holes, overlaps, and cycles.
3011      * @return false if there are errors
3012      * @throws IOException
3013      */
3014     public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
3015       // When table is disabled no need to check for the region chain. Some of the regions
3016       // accidently if deployed, this below code might report some issues like missing start
3017       // or end regions or region hole in chain and may try to fix which is unwanted.
3018       if (isTableDisabled(this.tableName)) {
3019         return true;
3020       }
3021       int originalErrorsCount = errors.getErrorList().size();
3022       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
3023       SortedSet<byte[]> splits = sc.getSplits();
3024 
3025       byte[] prevKey = null;
3026       byte[] problemKey = null;
3027 
3028       if (splits.size() == 0) {
3029         // no region for this table
3030         handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
3031       }
3032 
3033       for (byte[] key : splits) {
3034         Collection<HbckInfo> ranges = regions.get(key);
3035         if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
3036           for (HbckInfo rng : ranges) {
3037             handler.handleRegionStartKeyNotEmpty(rng);
3038           }
3039         }
3040 
3041         // check for degenerate ranges
3042         for (HbckInfo rng : ranges) {
3043           // special endkey case converts '' to null
3044           byte[] endKey = rng.getEndKey();
3045           endKey = (endKey.length == 0) ? null : endKey;
3046           if (Bytes.equals(rng.getStartKey(),endKey)) {
3047             handler.handleDegenerateRegion(rng);
3048           }
3049         }
3050 
3051         if (ranges.size() == 1) {
3052           // this split key is ok -- no overlap, not a hole.
3053           if (problemKey != null) {
3054             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3055           }
3056           problemKey = null; // fell through, no more problem.
3057         } else if (ranges.size() > 1) {
3058           // set the new problem key group name, if already have problem key, just
3059           // keep using it.
3060           if (problemKey == null) {
3061             // only for overlap regions.
3062             LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
3063             problemKey = key;
3064           }
3065           overlapGroups.putAll(problemKey, ranges);
3066 
3067           // record errors
3068           ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
3069           //  this dumb and n^2 but this shouldn't happen often
3070           for (HbckInfo r1 : ranges) {
3071             if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3072             subRange.remove(r1);
3073             for (HbckInfo r2 : subRange) {
3074               if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3075               if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
3076                 handler.handleDuplicateStartKeys(r1,r2);
3077               } else {
3078                 // overlap
3079                 handler.handleOverlapInRegionChain(r1, r2);
3080               }
3081             }
3082           }
3083 
3084         } else if (ranges.size() == 0) {
3085           if (problemKey != null) {
3086             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3087           }
3088           problemKey = null;
3089 
3090           byte[] holeStopKey = sc.getSplits().higher(key);
3091           // if higher key is null we reached the top.
3092           if (holeStopKey != null) {
3093             // hole
3094             handler.handleHoleInRegionChain(key, holeStopKey);
3095           }
3096         }
3097         prevKey = key;
3098       }
3099 
3100       // When the last region of a table is proper and having an empty end key, 'prevKey'
3101       // will be null.
3102       if (prevKey != null) {
3103         handler.handleRegionEndKeyNotEmpty(prevKey);
3104       }
3105 
3106       // TODO fold this into the TableIntegrityHandler
3107       if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
3108         boolean ok = handleOverlapsParallel(handler, prevKey);
3109         if (!ok) {
3110           return false;
3111         }
3112       } else {
3113         for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3114           handler.handleOverlapGroup(overlap);
3115         }
3116       }
3117 
3118       if (details) {
3119         // do full region split map dump
3120         errors.print("---- Table '"  +  this.tableName
3121             + "': region split map");
3122         dump(splits, regions);
3123         errors.print("---- Table '"  +  this.tableName
3124             + "': overlap groups");
3125         dumpOverlapProblems(overlapGroups);
3126         errors.print("There are " + overlapGroups.keySet().size()
3127             + " overlap groups with " + overlapGroups.size()
3128             + " overlapping regions");
3129       }
3130       if (!sidelinedRegions.isEmpty()) {
3131         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
3132         errors.print("---- Table '"  +  this.tableName
3133             + "': sidelined big overlapped regions");
3134         dumpSidelinedRegions(sidelinedRegions);
3135       }
3136       return errors.getErrorList().size() == originalErrorsCount;
3137     }
3138 
3139     private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
3140         throws IOException {
3141       // we parallelize overlap handler for the case we have lots of groups to fix.  We can
3142       // safely assume each group is independent.
3143       List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
3144       List<Future<Void>> rets;
3145       for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3146         //
3147         merges.add(new WorkItemOverlapMerge(overlap, handler));
3148       }
3149       try {
3150         rets = executor.invokeAll(merges);
3151       } catch (InterruptedException e) {
3152         LOG.error("Overlap merges were interrupted", e);
3153         return false;
3154       }
3155       for(int i=0; i<merges.size(); i++) {
3156         WorkItemOverlapMerge work = merges.get(i);
3157         Future<Void> f = rets.get(i);
3158         try {
3159           f.get();
3160         } catch(ExecutionException e) {
3161           LOG.warn("Failed to merge overlap group" + work, e.getCause());
3162         } catch (InterruptedException e) {
3163           LOG.error("Waiting for overlap merges was interrupted", e);
3164           return false;
3165         }
3166       }
3167       return true;
3168     }
3169 
3170     /**
3171      * This dumps data in a visually reasonable way for visual debugging
3172      *
3173      * @param splits
3174      * @param regions
3175      */
3176     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
3177       // we display this way because the last end key should be displayed as well.
3178       StringBuilder sb = new StringBuilder();
3179       for (byte[] k : splits) {
3180         sb.setLength(0); // clear out existing buffer, if any.
3181         sb.append(Bytes.toStringBinary(k) + ":\t");
3182         for (HbckInfo r : regions.get(k)) {
3183           sb.append("[ "+ r.toString() + ", "
3184               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
3185         }
3186         errors.print(sb.toString());
3187       }
3188     }
3189   }
3190 
3191   public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
3192     // we display this way because the last end key should be displayed as
3193     // well.
3194     for (byte[] k : regions.keySet()) {
3195       errors.print(Bytes.toStringBinary(k) + ":");
3196       for (HbckInfo r : regions.get(k)) {
3197         errors.print("[ " + r.toString() + ", "
3198             + Bytes.toStringBinary(r.getEndKey()) + "]");
3199       }
3200       errors.print("----");
3201     }
3202   }
3203 
3204   public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
3205     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
3206       TableName tableName = entry.getValue().getTableName();
3207       Path path = entry.getKey();
3208       errors.print("This sidelined region dir should be bulk loaded: "
3209         + path.toString());
3210       errors.print("Bulk load command looks like: "
3211         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
3212         + path.toUri().getPath() + " "+ tableName);
3213     }
3214   }
3215 
3216   public Multimap<byte[], HbckInfo> getOverlapGroups(
3217       TableName table) {
3218     TableInfo ti = tablesInfo.get(table);
3219     return ti.overlapGroups;
3220   }
3221 
3222   /**
3223    * Return a list of user-space table names whose metadata have not been
3224    * modified in the last few milliseconds specified by timelag
3225    * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
3226    * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
3227    * milliseconds specified by timelag, then the table is a candidate to be returned.
3228    * @return tables that have not been modified recently
3229    * @throws IOException if an error is encountered
3230    */
3231   HTableDescriptor[] getTables(AtomicInteger numSkipped) {
3232     List<TableName> tableNames = new ArrayList<TableName>();
3233     long now = EnvironmentEdgeManager.currentTime();
3234 
3235     for (HbckInfo hbi : regionInfoMap.values()) {
3236       MetaEntry info = hbi.metaEntry;
3237 
3238       // if the start key is zero, then we have found the first region of a table.
3239       // pick only those tables that were not modified in the last few milliseconds.
3240       if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
3241         if (info.modTime + timelag < now) {
3242           tableNames.add(info.getTable());
3243         } else {
3244           numSkipped.incrementAndGet(); // one more in-flux table
3245         }
3246       }
3247     }
3248     return getHTableDescriptors(tableNames);
3249   }
3250 
3251   HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
3252     HTableDescriptor[] htd = new HTableDescriptor[0];
3253       LOG.info("getHTableDescriptors == tableNames => " + tableNames);
3254     try (Connection conn = ConnectionFactory.createConnection(getConf());
3255         Admin admin = conn.getAdmin()) {
3256       htd = admin.getTableDescriptorsByTableName(tableNames);
3257     } catch (IOException e) {
3258       LOG.debug("Exception getting table descriptors", e);
3259     }
3260     return htd;
3261   }
3262 
3263   /**
3264    * Gets the entry in regionInfo corresponding to the the given encoded
3265    * region name. If the region has not been seen yet, a new entry is added
3266    * and returned.
3267    */
3268   private synchronized HbckInfo getOrCreateInfo(String name) {
3269     HbckInfo hbi = regionInfoMap.get(name);
3270     if (hbi == null) {
3271       hbi = new HbckInfo(null);
3272       regionInfoMap.put(name, hbi);
3273     }
3274     return hbi;
3275   }
3276 
3277   private void checkAndFixTableLocks() throws IOException {
3278     TableLockChecker checker = new TableLockChecker(zkw, errors);
3279     checker.checkTableLocks();
3280 
3281     if (this.fixTableLocks) {
3282       checker.fixExpiredTableLocks();
3283     }
3284   }
3285   
3286   private void checkAndFixReplication() throws IOException {
3287     ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, connection, errors);
3288     checker.checkUnDeletedQueues();
3289 
3290     if (checker.hasUnDeletedQueues() && this.fixReplication) {
3291       checker.fixUnDeletedQueues();
3292       setShouldRerun();
3293     }
3294   }
3295 
3296   /**
3297     * Check values in regionInfo for hbase:meta
3298     * Check if zero or more than one regions with hbase:meta are found.
3299     * If there are inconsistencies (i.e. zero or more than one regions
3300     * pretend to be holding the hbase:meta) try to fix that and report an error.
3301     * @throws IOException from HBaseFsckRepair functions
3302     * @throws KeeperException
3303     * @throws InterruptedException
3304     */
3305   boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
3306     Map<Integer, HbckInfo> metaRegions = new HashMap<Integer, HbckInfo>();
3307     for (HbckInfo value : regionInfoMap.values()) {
3308       if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
3309         metaRegions.put(value.getReplicaId(), value);
3310       }
3311     }
3312     int metaReplication = admin.getTableDescriptor(TableName.META_TABLE_NAME)
3313         .getRegionReplication();
3314     boolean noProblem = true;
3315     // There will be always entries in regionInfoMap corresponding to hbase:meta & its replicas
3316     // Check the deployed servers. It should be exactly one server for each replica.
3317     for (int i = 0; i < metaReplication; i++) {
3318       HbckInfo metaHbckInfo = metaRegions.remove(i);
3319       List<ServerName> servers = new ArrayList<ServerName>();
3320       if (metaHbckInfo != null) {
3321         servers = metaHbckInfo.deployedOn;
3322       }
3323       if (servers.size() != 1) {
3324         noProblem = false;
3325         if (servers.size() == 0) {
3326           assignMetaReplica(i);
3327         } else if (servers.size() > 1) {
3328           errors
3329           .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId " +
3330                        metaHbckInfo.getReplicaId() + " is found on more than one region.");
3331           if (shouldFixAssignments()) {
3332             errors.print("Trying to fix a problem with hbase:meta, replicaId " +
3333                          metaHbckInfo.getReplicaId() +"..");
3334             setShouldRerun();
3335             // try fix it (treat is a dupe assignment)
3336             HBaseFsckRepair.fixMultiAssignment(connection, metaHbckInfo.metaEntry, servers);
3337           }
3338         }
3339       }
3340     }
3341     // unassign whatever is remaining in metaRegions. They are excess replicas.
3342     for (Map.Entry<Integer, HbckInfo> entry : metaRegions.entrySet()) {
3343       noProblem = false;
3344       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
3345           "hbase:meta replicas are deployed in excess. Configured " + metaReplication +
3346           ", deployed " + metaRegions.size());
3347       if (shouldFixAssignments()) {
3348         errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() +
3349             " of hbase:meta..");
3350         setShouldRerun();
3351         unassignMetaReplica(entry.getValue());
3352       }
3353     }
3354     // if noProblem is false, rerun hbck with hopefully fixed META
3355     // if noProblem is true, no errors, so continue normally
3356     return noProblem;
3357   }
3358 
3359   private void unassignMetaReplica(HbckInfo hi) throws IOException, InterruptedException,
3360   KeeperException {
3361     undeployRegions(hi);
3362     ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(hi.metaEntry.getReplicaId()));
3363   }
3364 
3365   private void assignMetaReplica(int replicaId)
3366       throws IOException, KeeperException, InterruptedException {
3367     errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " +
3368         replicaId +" is not found on any region.");
3369     if (shouldFixAssignments()) {
3370       errors.print("Trying to fix a problem with hbase:meta..");
3371       setShouldRerun();
3372       // try to fix it (treat it as unassigned region)
3373       HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
3374           HRegionInfo.FIRST_META_REGIONINFO, replicaId);
3375       HBaseFsckRepair.fixUnassigned(admin, h);
3376       HBaseFsckRepair.waitUntilAssigned(admin, h);
3377     }
3378   }
3379 
3380   /**
3381    * Scan hbase:meta, adding all regions found to the regionInfo map.
3382    * @throws IOException if an error is encountered
3383    */
3384   boolean loadMetaEntries() throws IOException {
3385     MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
3386       int countRecord = 1;
3387 
3388       // comparator to sort KeyValues with latest modtime
3389       final Comparator<Cell> comp = new Comparator<Cell>() {
3390         @Override
3391         public int compare(Cell k1, Cell k2) {
3392           return (int)(k1.getTimestamp() - k2.getTimestamp());
3393         }
3394       };
3395 
3396       @Override
3397       public boolean visit(Result result) throws IOException {
3398         try {
3399 
3400           // record the latest modification of this META record
3401           long ts =  Collections.max(result.listCells(), comp).getTimestamp();
3402           RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
3403           if (rl == null) {
3404             emptyRegionInfoQualifiers.add(result);
3405             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3406               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3407             return true;
3408           }
3409           ServerName sn = null;
3410           if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
3411               rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
3412             emptyRegionInfoQualifiers.add(result);
3413             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3414               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3415             return true;
3416           }
3417           HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
3418           if (!(isTableIncluded(hri.getTable())
3419               || hri.isMetaRegion())) {
3420             return true;
3421           }
3422           PairOfSameType<HRegionInfo> daughters = MetaTableAccessor.getDaughterRegions(result);
3423           for (HRegionLocation h : rl.getRegionLocations()) {
3424             if (h == null || h.getRegionInfo() == null) {
3425               continue;
3426             }
3427             sn = h.getServerName();
3428             hri = h.getRegionInfo();
3429 
3430             MetaEntry m = null;
3431             if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
3432               m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
3433             } else {
3434               m = new MetaEntry(hri, sn, ts, null, null);
3435             }
3436             HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
3437             if (previous == null) {
3438               regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
3439             } else if (previous.metaEntry == null) {
3440               previous.metaEntry = m;
3441             } else {
3442               throw new IOException("Two entries in hbase:meta are same " + previous);
3443             }
3444           }
3445           PairOfSameType<HRegionInfo> mergeRegions = MetaTableAccessor.getMergeRegions(result);
3446           for (HRegionInfo mergeRegion : new HRegionInfo[] {
3447               mergeRegions.getFirst(), mergeRegions.getSecond() }) {
3448             if (mergeRegion != null) {
3449               // This region is already been merged
3450               HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
3451               hbInfo.setMerged(true);
3452             }
3453           }
3454 
3455           // show proof of progress to the user, once for every 100 records.
3456           if (countRecord % 100 == 0) {
3457             errors.progress();
3458           }
3459           countRecord++;
3460           return true;
3461         } catch (RuntimeException e) {
3462           LOG.error("Result=" + result);
3463           throw e;
3464         }
3465       }
3466     };
3467     if (!checkMetaOnly) {
3468       // Scan hbase:meta to pick up user regions
3469       MetaTableAccessor.fullScanRegions(connection, visitor);
3470     }
3471 
3472     errors.print("");
3473     return true;
3474   }
3475 
3476   /**
3477    * Stores the regioninfo entries scanned from META
3478    */
3479   static class MetaEntry extends HRegionInfo {
3480     ServerName regionServer;   // server hosting this region
3481     long modTime;          // timestamp of most recent modification metadata
3482     HRegionInfo splitA, splitB; //split daughters
3483 
3484     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3485       this(rinfo, regionServer, modTime, null, null);
3486     }
3487 
3488     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3489         HRegionInfo splitA, HRegionInfo splitB) {
3490       super(rinfo);
3491       this.regionServer = regionServer;
3492       this.modTime = modTime;
3493       this.splitA = splitA;
3494       this.splitB = splitB;
3495     }
3496 
3497     @Override
3498     public boolean equals(Object o) {
3499       boolean superEq = super.equals(o);
3500       if (!superEq) {
3501         return superEq;
3502       }
3503 
3504       MetaEntry me = (MetaEntry) o;
3505       if (!regionServer.equals(me.regionServer)) {
3506         return false;
3507       }
3508       return (modTime == me.modTime);
3509     }
3510 
3511     @Override
3512     public int hashCode() {
3513       int hash = Arrays.hashCode(getRegionName());
3514       hash ^= getRegionId();
3515       hash ^= Arrays.hashCode(getStartKey());
3516       hash ^= Arrays.hashCode(getEndKey());
3517       hash ^= Boolean.valueOf(isOffline()).hashCode();
3518       hash ^= getTable().hashCode();
3519       if (regionServer != null) {
3520         hash ^= regionServer.hashCode();
3521       }
3522       hash ^= modTime;
3523       return hash;
3524     }
3525   }
3526 
3527   /**
3528    * Stores the regioninfo entries from HDFS
3529    */
3530   static class HdfsEntry {
3531     HRegionInfo hri;
3532     Path hdfsRegionDir = null;
3533     long hdfsRegionDirModTime  = 0;
3534     boolean hdfsRegioninfoFilePresent = false;
3535     boolean hdfsOnlyEdits = false;
3536   }
3537 
3538   /**
3539    * Stores the regioninfo retrieved from Online region servers.
3540    */
3541   static class OnlineEntry {
3542     HRegionInfo hri;
3543     ServerName hsa;
3544 
3545     @Override
3546     public String toString() {
3547       return hsa.toString() + ";" + hri.getRegionNameAsString();
3548     }
3549   }
3550 
3551   /**
3552    * Maintain information about a particular region.  It gathers information
3553    * from three places -- HDFS, META, and region servers.
3554    */
3555   public static class HbckInfo implements KeyRange {
3556     private MetaEntry metaEntry = null; // info in META
3557     private HdfsEntry hdfsEntry = null; // info in HDFS
3558     private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
3559     private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
3560     private boolean skipChecks = false; // whether to skip further checks to this region info.
3561     private boolean isMerged = false;// whether this region has already been merged into another one
3562     private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3563     private HRegionInfo primaryHRIForDeployedReplica = null;
3564 
3565     HbckInfo(MetaEntry metaEntry) {
3566       this.metaEntry = metaEntry;
3567     }
3568 
3569     public synchronized int getReplicaId() {
3570       return metaEntry != null? metaEntry.getReplicaId(): deployedReplicaId;
3571     }
3572 
3573     public synchronized void addServer(HRegionInfo hri, ServerName server) {
3574       OnlineEntry rse = new OnlineEntry() ;
3575       rse.hri = hri;
3576       rse.hsa = server;
3577       this.deployedEntries.add(rse);
3578       this.deployedOn.add(server);
3579       // save the replicaId that we see deployed in the cluster
3580       this.deployedReplicaId = hri.getReplicaId();
3581       this.primaryHRIForDeployedReplica =
3582           RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3583     }
3584 
3585     @Override
3586     public synchronized String toString() {
3587       StringBuilder sb = new StringBuilder();
3588       sb.append("{ meta => ");
3589       sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3590       sb.append( ", hdfs => " + getHdfsRegionDir());
3591       sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3592       sb.append( ", replicaId => " + getReplicaId());
3593       sb.append(" }");
3594       return sb.toString();
3595     }
3596 
3597     @Override
3598     public byte[] getStartKey() {
3599       if (this.metaEntry != null) {
3600         return this.metaEntry.getStartKey();
3601       } else if (this.hdfsEntry != null) {
3602         return this.hdfsEntry.hri.getStartKey();
3603       } else {
3604         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3605         return null;
3606       }
3607     }
3608 
3609     @Override
3610     public byte[] getEndKey() {
3611       if (this.metaEntry != null) {
3612         return this.metaEntry.getEndKey();
3613       } else if (this.hdfsEntry != null) {
3614         return this.hdfsEntry.hri.getEndKey();
3615       } else {
3616         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3617         return null;
3618       }
3619     }
3620 
3621     public TableName getTableName() {
3622       if (this.metaEntry != null) {
3623         return this.metaEntry.getTable();
3624       } else if (this.hdfsEntry != null) {
3625         // we are only guaranteed to have a path and not an HRI for hdfsEntry,
3626         // so we get the name from the Path
3627         Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3628         return FSUtils.getTableName(tableDir);
3629       } else {
3630         // return the info from the first online/deployed hri
3631         for (OnlineEntry e : deployedEntries) {
3632           return e.hri.getTable();
3633         }
3634         return null;
3635       }
3636     }
3637 
3638     public String getRegionNameAsString() {
3639       if (metaEntry != null) {
3640         return metaEntry.getRegionNameAsString();
3641       } else if (hdfsEntry != null) {
3642         if (hdfsEntry.hri != null) {
3643           return hdfsEntry.hri.getRegionNameAsString();
3644         }
3645       } else {
3646         // return the info from the first online/deployed hri
3647         for (OnlineEntry e : deployedEntries) {
3648           return e.hri.getRegionNameAsString();
3649         }
3650       }
3651       return null;
3652     }
3653 
3654     public byte[] getRegionName() {
3655       if (metaEntry != null) {
3656         return metaEntry.getRegionName();
3657       } else if (hdfsEntry != null) {
3658         return hdfsEntry.hri.getRegionName();
3659       } else {
3660         // return the info from the first online/deployed hri
3661         for (OnlineEntry e : deployedEntries) {
3662           return e.hri.getRegionName();
3663         }
3664         return null;
3665       }
3666     }
3667 
3668     public HRegionInfo getPrimaryHRIForDeployedReplica() {
3669       return primaryHRIForDeployedReplica;
3670     }
3671 
3672     Path getHdfsRegionDir() {
3673       if (hdfsEntry == null) {
3674         return null;
3675       }
3676       return hdfsEntry.hdfsRegionDir;
3677     }
3678 
3679     boolean containsOnlyHdfsEdits() {
3680       if (hdfsEntry == null) {
3681         return false;
3682       }
3683       return hdfsEntry.hdfsOnlyEdits;
3684     }
3685 
3686     boolean isHdfsRegioninfoPresent() {
3687       if (hdfsEntry == null) {
3688         return false;
3689       }
3690       return hdfsEntry.hdfsRegioninfoFilePresent;
3691     }
3692 
3693     long getModTime() {
3694       if (hdfsEntry == null) {
3695         return 0;
3696       }
3697       return hdfsEntry.hdfsRegionDirModTime;
3698     }
3699 
3700     HRegionInfo getHdfsHRI() {
3701       if (hdfsEntry == null) {
3702         return null;
3703       }
3704       return hdfsEntry.hri;
3705     }
3706 
3707     public void setSkipChecks(boolean skipChecks) {
3708       this.skipChecks = skipChecks;
3709     }
3710 
3711     public boolean isSkipChecks() {
3712       return skipChecks;
3713     }
3714 
3715     public void setMerged(boolean isMerged) {
3716       this.isMerged = isMerged;
3717     }
3718 
3719     public boolean isMerged() {
3720       return this.isMerged;
3721     }
3722   }
3723 
3724   final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3725     @Override
3726     public int compare(HbckInfo l, HbckInfo r) {
3727       if (l == r) {
3728         // same instance
3729         return 0;
3730       }
3731 
3732       int tableCompare = l.getTableName().compareTo(r.getTableName());
3733       if (tableCompare != 0) {
3734         return tableCompare;
3735       }
3736 
3737       int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3738           l.getStartKey(), r.getStartKey());
3739       if (startComparison != 0) {
3740         return startComparison;
3741       }
3742 
3743       // Special case for absolute endkey
3744       byte[] endKey = r.getEndKey();
3745       endKey = (endKey.length == 0) ? null : endKey;
3746       byte[] endKey2 = l.getEndKey();
3747       endKey2 = (endKey2.length == 0) ? null : endKey2;
3748       int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3749           endKey2,  endKey);
3750 
3751       if (endComparison != 0) {
3752         return endComparison;
3753       }
3754 
3755       // use regionId as tiebreaker.
3756       // Null is considered after all possible values so make it bigger.
3757       if (l.hdfsEntry == null && r.hdfsEntry == null) {
3758         return 0;
3759       }
3760       if (l.hdfsEntry == null && r.hdfsEntry != null) {
3761         return 1;
3762       }
3763       // l.hdfsEntry must not be null
3764       if (r.hdfsEntry == null) {
3765         return -1;
3766       }
3767       // both l.hdfsEntry and r.hdfsEntry must not be null.
3768       return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3769     }
3770   };
3771 
3772   /**
3773    * Prints summary of all tables found on the system.
3774    */
3775   private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3776     StringBuilder sb = new StringBuilder();
3777     int numOfSkippedRegions;
3778     errors.print("Summary:");
3779     for (TableInfo tInfo : tablesInfo.values()) {
3780       numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName())) ?
3781           skippedRegions.get(tInfo.getName()).size() : 0;
3782 
3783       if (errors.tableHasErrors(tInfo)) {
3784         errors.print("Table " + tInfo.getName() + " is inconsistent.");
3785       } else if (numOfSkippedRegions > 0){
3786         errors.print("Table " + tInfo.getName() + " is okay (with "
3787           + numOfSkippedRegions + " skipped regions).");
3788       }
3789       else {
3790         errors.print("Table " + tInfo.getName() + " is okay.");
3791       }
3792       errors.print("    Number of regions: " + tInfo.getNumRegions());
3793       if (numOfSkippedRegions > 0) {
3794         Set<String> skippedRegionStrings = skippedRegions.get(tInfo.getName());
3795         System.out.println("    Number of skipped regions: " + numOfSkippedRegions);
3796         System.out.println("      List of skipped regions:");
3797         for(String sr : skippedRegionStrings) {
3798           System.out.println("        " + sr);
3799         }
3800       }
3801       sb.setLength(0); // clear out existing buffer, if any.
3802       sb.append("    Deployed on: ");
3803       for (ServerName server : tInfo.deployedOn) {
3804         sb.append(" " + server.toString());
3805       }
3806       errors.print(sb.toString());
3807     }
3808   }
3809 
3810   static ErrorReporter getErrorReporter(
3811       final Configuration conf) throws ClassNotFoundException {
3812     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3813     return ReflectionUtils.newInstance(reporter, conf);
3814   }
3815 
3816   public interface ErrorReporter {
3817     enum ERROR_CODE {
3818       UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3819       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META,
3820       NOT_DEPLOYED,
3821       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3822       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3823       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3824       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3825       WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR, ORPHAN_TABLE_STATE,
3826       NO_TABLE_STATE, UNDELETED_REPLICATION_QUEUE
3827     }
3828     void clear();
3829     void report(String message);
3830     void reportError(String message);
3831     void reportError(ERROR_CODE errorCode, String message);
3832     void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3833     void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3834     void reportError(
3835       ERROR_CODE errorCode,
3836       String message,
3837       TableInfo table,
3838       HbckInfo info1,
3839       HbckInfo info2
3840     );
3841     int summarize();
3842     void detail(String details);
3843     ArrayList<ERROR_CODE> getErrorList();
3844     void progress();
3845     void print(String message);
3846     void resetErrors();
3847     boolean tableHasErrors(TableInfo table);
3848   }
3849 
3850   static class PrintingErrorReporter implements ErrorReporter {
3851     public int errorCount = 0;
3852     private int showProgress;
3853     // How frequently calls to progress() will create output
3854     private static final int progressThreshold = 100;
3855 
3856     Set<TableInfo> errorTables = new HashSet<TableInfo>();
3857 
3858     // for use by unit tests to verify which errors were discovered
3859     private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3860 
3861     @Override
3862     public void clear() {
3863       errorTables.clear();
3864       errorList.clear();
3865       errorCount = 0;
3866     }
3867 
3868     @Override
3869     public synchronized void reportError(ERROR_CODE errorCode, String message) {
3870       if (errorCode == ERROR_CODE.WRONG_USAGE) {
3871         System.err.println(message);
3872         return;
3873       }
3874 
3875       errorList.add(errorCode);
3876       if (!summary) {
3877         System.out.println("ERROR: " + message);
3878       }
3879       errorCount++;
3880       showProgress = 0;
3881     }
3882 
3883     @Override
3884     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
3885       errorTables.add(table);
3886       reportError(errorCode, message);
3887     }
3888 
3889     @Override
3890     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3891                                          HbckInfo info) {
3892       errorTables.add(table);
3893       String reference = "(region " + info.getRegionNameAsString() + ")";
3894       reportError(errorCode, reference + " " + message);
3895     }
3896 
3897     @Override
3898     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3899                                          HbckInfo info1, HbckInfo info2) {
3900       errorTables.add(table);
3901       String reference = "(regions " + info1.getRegionNameAsString()
3902           + " and " + info2.getRegionNameAsString() + ")";
3903       reportError(errorCode, reference + " " + message);
3904     }
3905 
3906     @Override
3907     public synchronized void reportError(String message) {
3908       reportError(ERROR_CODE.UNKNOWN, message);
3909     }
3910 
3911     /**
3912      * Report error information, but do not increment the error count.  Intended for cases
3913      * where the actual error would have been reported previously.
3914      * @param message
3915      */
3916     @Override
3917     public synchronized void report(String message) {
3918       if (! summary) {
3919         System.out.println("ERROR: " + message);
3920       }
3921       showProgress = 0;
3922     }
3923 
3924     @Override
3925     public synchronized int summarize() {
3926       System.out.println(Integer.toString(errorCount) +
3927                          " inconsistencies detected.");
3928       if (errorCount == 0) {
3929         System.out.println("Status: OK");
3930         return 0;
3931       } else {
3932         System.out.println("Status: INCONSISTENT");
3933         return -1;
3934       }
3935     }
3936 
3937     @Override
3938     public ArrayList<ERROR_CODE> getErrorList() {
3939       return errorList;
3940     }
3941 
3942     @Override
3943     public synchronized void print(String message) {
3944       if (!summary) {
3945         System.out.println(message);
3946       }
3947     }
3948 
3949     @Override
3950     public boolean tableHasErrors(TableInfo table) {
3951       return errorTables.contains(table);
3952     }
3953 
3954     @Override
3955     public void resetErrors() {
3956       errorCount = 0;
3957     }
3958 
3959     @Override
3960     public synchronized void detail(String message) {
3961       if (details) {
3962         System.out.println(message);
3963       }
3964       showProgress = 0;
3965     }
3966 
3967     @Override
3968     public synchronized void progress() {
3969       if (showProgress++ == progressThreshold) {
3970         if (!summary) {
3971           System.out.print(".");
3972         }
3973         showProgress = 0;
3974       }
3975     }
3976   }
3977 
3978   /**
3979    * Contact a region server and get all information from it
3980    */
3981   static class WorkItemRegion implements Callable<Void> {
3982     private HBaseFsck hbck;
3983     private ServerName rsinfo;
3984     private ErrorReporter errors;
3985     private HConnection connection;
3986 
3987     WorkItemRegion(HBaseFsck hbck, ServerName info,
3988                    ErrorReporter errors, HConnection connection) {
3989       this.hbck = hbck;
3990       this.rsinfo = info;
3991       this.errors = errors;
3992       this.connection = connection;
3993     }
3994 
3995     @Override
3996     public synchronized Void call() throws IOException {
3997       errors.progress();
3998       try {
3999         BlockingInterface server = connection.getAdmin(rsinfo);
4000 
4001         // list all online regions from this region server
4002         List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
4003         regions = filterRegions(regions);
4004 
4005         if (details) {
4006           errors.detail("RegionServer: " + rsinfo.getServerName() +
4007                            " number of regions: " + regions.size());
4008           for (HRegionInfo rinfo: regions) {
4009             errors.detail("  " + rinfo.getRegionNameAsString() +
4010                              " id: " + rinfo.getRegionId() +
4011                              " encoded_name: " + rinfo.getEncodedName() +
4012                              " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
4013                              " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
4014           }
4015         }
4016 
4017         // check to see if the existence of this region matches the region in META
4018         for (HRegionInfo r:regions) {
4019           HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
4020           hbi.addServer(r, rsinfo);
4021         }
4022       } catch (IOException e) {          // unable to connect to the region server.
4023         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
4024           " Unable to fetch region information. " + e);
4025         throw e;
4026       }
4027       return null;
4028     }
4029 
4030     private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
4031       List<HRegionInfo> ret = Lists.newArrayList();
4032       for (HRegionInfo hri : regions) {
4033         if (hri.isMetaTable() || (!hbck.checkMetaOnly
4034             && hbck.isTableIncluded(hri.getTable()))) {
4035           ret.add(hri);
4036         }
4037       }
4038       return ret;
4039     }
4040   }
4041 
4042   /**
4043    * Contact hdfs and get all information about specified table directory into
4044    * regioninfo list.
4045    */
4046   static class WorkItemHdfsDir implements Callable<Void> {
4047     private HBaseFsck hbck;
4048     private FileStatus tableDir;
4049     private ErrorReporter errors;
4050     private FileSystem fs;
4051 
4052     WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
4053                     FileStatus status) {
4054       this.hbck = hbck;
4055       this.fs = fs;
4056       this.tableDir = status;
4057       this.errors = errors;
4058     }
4059 
4060     @Override
4061     public synchronized Void call() throws IOException {
4062       try {
4063         // level 2: <HBASE_DIR>/<table>/*
4064         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
4065         for (FileStatus regionDir : regionDirs) {
4066           errors.progress();
4067           String encodedName = regionDir.getPath().getName();
4068           // ignore directories that aren't hexadecimal
4069           if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
4070             continue;
4071           }
4072 
4073           LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
4074           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
4075           HdfsEntry he = new HdfsEntry();
4076           synchronized (hbi) {
4077             if (hbi.getHdfsRegionDir() != null) {
4078               errors.print("Directory " + encodedName + " duplicate??" +
4079                            hbi.getHdfsRegionDir());
4080             }
4081 
4082             he.hdfsRegionDir = regionDir.getPath();
4083             he.hdfsRegionDirModTime = regionDir.getModificationTime();
4084             Path regioninfoFile = new Path(he.hdfsRegionDir, HRegionFileSystem.REGION_INFO_FILE);
4085             he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
4086             // we add to orphan list when we attempt to read .regioninfo
4087 
4088             // Set a flag if this region contains only edits
4089             // This is special case if a region is left after split
4090             he.hdfsOnlyEdits = true;
4091             FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
4092             Path ePath = WALSplitter.getRegionDirRecoveredEditsDir(regionDir.getPath());
4093             for (FileStatus subDir : subDirs) {
4094               errors.progress();
4095               String sdName = subDir.getPath().getName();
4096               if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
4097                 he.hdfsOnlyEdits = false;
4098                 break;
4099               }
4100             }
4101             hbi.hdfsEntry = he;
4102           }
4103         }
4104       } catch (IOException e) {
4105         // unable to connect to the region server.
4106         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
4107             + tableDir.getPath().getName()
4108             + " Unable to fetch region information. " + e);
4109         throw e;
4110       }
4111       return null;
4112     }
4113   }
4114 
4115   /**
4116    * Contact hdfs and get all information about specified table directory into
4117    * regioninfo list.
4118    */
4119   static class WorkItemHdfsRegionInfo implements Callable<Void> {
4120     private HbckInfo hbi;
4121     private HBaseFsck hbck;
4122     private ErrorReporter errors;
4123 
4124     WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
4125       this.hbi = hbi;
4126       this.hbck = hbck;
4127       this.errors = errors;
4128     }
4129 
4130     @Override
4131     public synchronized Void call() throws IOException {
4132       // only load entries that haven't been loaded yet.
4133       if (hbi.getHdfsHRI() == null) {
4134         try {
4135           errors.progress();
4136           hbck.loadHdfsRegioninfo(hbi);
4137         } catch (IOException ioe) {
4138           String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
4139               + hbi.getTableName() + " in hdfs dir "
4140               + hbi.getHdfsRegionDir()
4141               + "!  It may be an invalid format or version file.  Treating as "
4142               + "an orphaned regiondir.";
4143           errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
4144           try {
4145             hbck.debugLsr(hbi.getHdfsRegionDir());
4146           } catch (IOException ioe2) {
4147             LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
4148             throw ioe2;
4149           }
4150           hbck.orphanHdfsDirs.add(hbi);
4151           throw ioe;
4152         }
4153       }
4154       return null;
4155     }
4156   };
4157 
4158   /**
4159    * Display the full report from fsck. This displays all live and dead region
4160    * servers, and all known regions.
4161    */
4162   public static void setDisplayFullReport() {
4163     details = true;
4164   }
4165 
4166   /**
4167    * Set exclusive mode.
4168    */
4169   public static void setForceExclusive() {
4170     forceExclusive = true;
4171   }
4172 
4173   /**
4174    * Only one instance of hbck can modify HBase at a time.
4175    */
4176   public boolean isExclusive() {
4177     return fixAny || forceExclusive;
4178   }
4179 
4180   /**
4181    * Disable the load balancer.
4182    */
4183   public static void setDisableBalancer() {
4184     disableBalancer = true;
4185   }
4186 
4187   /**
4188    * Disable the split and merge
4189    */
4190   public static void setDisableSplitAndMerge() {
4191     setDisableSplitAndMerge(true);
4192   }
4193 
4194   @VisibleForTesting
4195   public static void setDisableSplitAndMerge(boolean flag) {
4196     disableSplitAndMerge = flag;
4197   }
4198 
4199   /**
4200    * The balancer should be disabled if we are modifying HBase.
4201    * It can be disabled if you want to prevent region movement from causing
4202    * false positives.
4203    */
4204   public boolean shouldDisableBalancer() {
4205     return fixAny || disableBalancer;
4206   }
4207 
4208   /**
4209    * The split and merge should be disabled if we are modifying HBase.
4210    * It can be disabled if you want to prevent region movement from causing
4211    * false positives.
4212    */
4213   public boolean shouldDisableSplitAndMerge() {
4214     return fixAny || disableSplitAndMerge;
4215   }
4216   
4217   /**
4218    * Set summary mode.
4219    * Print only summary of the tables and status (OK or INCONSISTENT)
4220    */
4221   static void setSummary() {
4222     summary = true;
4223   }
4224 
4225   /**
4226    * Set hbase:meta check mode.
4227    * Print only info about hbase:meta table deployment/state
4228    */
4229   void setCheckMetaOnly() {
4230     checkMetaOnly = true;
4231   }
4232 
4233   /**
4234    * Set region boundaries check mode.
4235    */
4236   void setRegionBoundariesCheck() {
4237     checkRegionBoundaries = true;
4238   }
4239 
4240   /**
4241    * Set table locks fix mode.
4242    * Delete table locks held for a long time
4243    */
4244   public void setFixTableLocks(boolean shouldFix) {
4245     fixTableLocks = shouldFix;
4246     fixAny |= shouldFix;
4247   }
4248   
4249   /**
4250    * Set replication fix mode.
4251    */
4252   public void setFixReplication(boolean shouldFix) {
4253     fixReplication = shouldFix;
4254     fixAny |= shouldFix;
4255   }
4256 
4257   /**
4258    * Check if we should rerun fsck again. This checks if we've tried to
4259    * fix something and we should rerun fsck tool again.
4260    * Display the full report from fsck. This displays all live and dead
4261    * region servers, and all known regions.
4262    */
4263   void setShouldRerun() {
4264     rerun = true;
4265   }
4266 
4267   boolean shouldRerun() {
4268     return rerun;
4269   }
4270 
4271   /**
4272    * Fix inconsistencies found by fsck. This should try to fix errors (if any)
4273    * found by fsck utility.
4274    */
4275   public void setFixAssignments(boolean shouldFix) {
4276     fixAssignments = shouldFix;
4277     fixAny |= shouldFix;
4278   }
4279 
4280   boolean shouldFixAssignments() {
4281     return fixAssignments;
4282   }
4283 
4284   public void setFixMeta(boolean shouldFix) {
4285     fixMeta = shouldFix;
4286     fixAny |= shouldFix;
4287   }
4288 
4289   boolean shouldFixMeta() {
4290     return fixMeta;
4291   }
4292 
4293   public void setFixEmptyMetaCells(boolean shouldFix) {
4294     fixEmptyMetaCells = shouldFix;
4295     fixAny |= shouldFix;
4296   }
4297 
4298   boolean shouldFixEmptyMetaCells() {
4299     return fixEmptyMetaCells;
4300   }
4301 
4302   public void setCheckHdfs(boolean checking) {
4303     checkHdfs = checking;
4304   }
4305 
4306   boolean shouldCheckHdfs() {
4307     return checkHdfs;
4308   }
4309 
4310   public void setFixHdfsHoles(boolean shouldFix) {
4311     fixHdfsHoles = shouldFix;
4312     fixAny |= shouldFix;
4313   }
4314 
4315   boolean shouldFixHdfsHoles() {
4316     return fixHdfsHoles;
4317   }
4318 
4319   public void setFixTableOrphans(boolean shouldFix) {
4320     fixTableOrphans = shouldFix;
4321     fixAny |= shouldFix;
4322   }
4323 
4324   boolean shouldFixTableOrphans() {
4325     return fixTableOrphans;
4326   }
4327 
4328   public void setFixHdfsOverlaps(boolean shouldFix) {
4329     fixHdfsOverlaps = shouldFix;
4330     fixAny |= shouldFix;
4331   }
4332 
4333   boolean shouldFixHdfsOverlaps() {
4334     return fixHdfsOverlaps;
4335   }
4336 
4337   public void setFixHdfsOrphans(boolean shouldFix) {
4338     fixHdfsOrphans = shouldFix;
4339     fixAny |= shouldFix;
4340   }
4341 
4342   boolean shouldFixHdfsOrphans() {
4343     return fixHdfsOrphans;
4344   }
4345 
4346   public void setFixVersionFile(boolean shouldFix) {
4347     fixVersionFile = shouldFix;
4348     fixAny |= shouldFix;
4349   }
4350 
4351   public boolean shouldFixVersionFile() {
4352     return fixVersionFile;
4353   }
4354 
4355   public void setSidelineBigOverlaps(boolean sbo) {
4356     this.sidelineBigOverlaps = sbo;
4357   }
4358 
4359   public boolean shouldSidelineBigOverlaps() {
4360     return sidelineBigOverlaps;
4361   }
4362 
4363   public void setFixSplitParents(boolean shouldFix) {
4364     fixSplitParents = shouldFix;
4365     fixAny |= shouldFix;
4366   }
4367 
4368   boolean shouldFixSplitParents() {
4369     return fixSplitParents;
4370   }
4371 
4372   public void setFixReferenceFiles(boolean shouldFix) {
4373     fixReferenceFiles = shouldFix;
4374     fixAny |= shouldFix;
4375   }
4376 
4377   boolean shouldFixReferenceFiles() {
4378     return fixReferenceFiles;
4379   }
4380 
4381   public boolean shouldIgnorePreCheckPermission() {
4382     return !fixAny || ignorePreCheckPermission;
4383   }
4384 
4385   public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
4386     this.ignorePreCheckPermission = ignorePreCheckPermission;
4387   }
4388 
4389   /**
4390    * @param mm maximum number of regions to merge into a single region.
4391    */
4392   public void setMaxMerge(int mm) {
4393     this.maxMerge = mm;
4394   }
4395 
4396   public int getMaxMerge() {
4397     return maxMerge;
4398   }
4399 
4400   public void setMaxOverlapsToSideline(int mo) {
4401     this.maxOverlapsToSideline = mo;
4402   }
4403 
4404   public int getMaxOverlapsToSideline() {
4405     return maxOverlapsToSideline;
4406   }
4407 
4408   /**
4409    * Only check/fix tables specified by the list,
4410    * Empty list means all tables are included.
4411    */
4412   boolean isTableIncluded(TableName table) {
4413     return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
4414   }
4415 
4416   public void includeTable(TableName table) {
4417     tablesIncluded.add(table);
4418   }
4419 
4420   Set<TableName> getIncludedTables() {
4421     return new HashSet<TableName>(tablesIncluded);
4422   }
4423 
4424   /**
4425    * We are interested in only those tables that have not changed their state in
4426    * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
4427    * @param seconds - the time in seconds
4428    */
4429   public void setTimeLag(long seconds) {
4430     timelag = seconds * 1000; // convert to milliseconds
4431   }
4432 
4433   /**
4434    *
4435    * @param sidelineDir - HDFS path to sideline data
4436    */
4437   public void setSidelineDir(String sidelineDir) {
4438     this.sidelineDir = new Path(sidelineDir);
4439   }
4440 
4441   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
4442     return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
4443   }
4444 
4445   public HFileCorruptionChecker getHFilecorruptionChecker() {
4446     return hfcc;
4447   }
4448 
4449   public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
4450     this.hfcc = hfcc;
4451   }
4452 
4453   public void setRetCode(int code) {
4454     this.retcode = code;
4455   }
4456 
4457   public int getRetCode() {
4458     return retcode;
4459   }
4460 
4461   protected HBaseFsck printUsageAndExit() {
4462     StringWriter sw = new StringWriter(2048);
4463     PrintWriter out = new PrintWriter(sw);
4464     out.println("Usage: fsck [opts] {only tables}");
4465     out.println(" where [opts] are:");
4466     out.println("   -help Display help options (this)");
4467     out.println("   -details Display full report of all regions.");
4468     out.println("   -timelag <timeInSeconds>  Process only regions that " +
4469                        " have not experienced any metadata updates in the last " +
4470                        " <timeInSeconds> seconds.");
4471     out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
4472         " before checking if the fix worked if run with -fix");
4473     out.println("   -summary Print only summary of the tables and status.");
4474     out.println("   -metaonly Only check the state of the hbase:meta table.");
4475     out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
4476     out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
4477     out.println("   -exclusive Abort if another hbck is exclusive or fixing.");
4478     out.println("   -disableBalancer Disable the load balancer.");
4479 
4480     out.println("");
4481     out.println("  Metadata Repair options: (expert features, use with caution!)");
4482     out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
4483     out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
4484     out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
4485     out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
4486         + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
4487     out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
4488     out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
4489     out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
4490     out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
4491     out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
4492     out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
4493     out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
4494     out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
4495     out.println("   -fixSplitParents  Try to force offline split parents to be online.");
4496     out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
4497     out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
4498     out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
4499         + " (empty REGIONINFO_QUALIFIER rows)");
4500 
4501     out.println("");
4502     out.println("  Datafile Repair options: (expert features, use with caution!)");
4503     out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
4504     out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
4505 
4506     out.println("");
4507     out.println("  Metadata Repair shortcuts");
4508     out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
4509         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles -fixTableLocks");
4510     out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
4511 
4512     out.println("");
4513     out.println("  Table lock options");
4514     out.println("   -fixTableLocks    Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
4515 
4516     out.println("");
4517     out.println(" Replication options");
4518     out.println("   -fixReplication   Deletes replication queues for removed peers");
4519     
4520     out.flush();
4521     errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
4522 
4523     setRetCode(-2);
4524     return this;
4525   }
4526 
4527   /**
4528    * Main program
4529    *
4530    * @param args
4531    * @throws Exception
4532    */
4533   public static void main(String[] args) throws Exception {
4534     // create a fsck object
4535     Configuration conf = HBaseConfiguration.create();
4536     Path hbasedir = FSUtils.getRootDir(conf);
4537     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
4538     FSUtils.setFsDefault(conf, new Path(defaultFs));
4539     int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
4540     System.exit(ret);
4541   }
4542 
4543   /**
4544    * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
4545    */
4546   static class HBaseFsckTool extends Configured implements Tool {
4547     HBaseFsckTool(Configuration conf) { super(conf); }
4548     @Override
4549     public int run(String[] args) throws Exception {
4550       HBaseFsck hbck = new HBaseFsck(getConf());
4551       hbck.exec(hbck.executor, args);
4552       hbck.close();
4553       return hbck.getRetCode();
4554     }
4555   };
4556 
4557 
4558   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
4559     ServiceException, InterruptedException {
4560     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
4561 
4562     boolean checkCorruptHFiles = false;
4563     boolean sidelineCorruptHFiles = false;
4564 
4565     // Process command-line args.
4566     for (int i = 0; i < args.length; i++) {
4567       String cmd = args[i];
4568       if (cmd.equals("-help") || cmd.equals("-h")) {
4569         return printUsageAndExit();
4570       } else if (cmd.equals("-details")) {
4571         setDisplayFullReport();
4572       } else if (cmd.equals("-exclusive")) {
4573         setForceExclusive();
4574       } else if (cmd.equals("-disableBalancer")) {
4575         setDisableBalancer();
4576       }  else if (cmd.equals("-disableSplitAndMerge")) {
4577         setDisableSplitAndMerge();
4578       } else if (cmd.equals("-timelag")) {
4579         if (i == args.length - 1) {
4580           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4581           return printUsageAndExit();
4582         }
4583         try {
4584           long timelag = Long.parseLong(args[i+1]);
4585           setTimeLag(timelag);
4586         } catch (NumberFormatException e) {
4587           errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4588           return printUsageAndExit();
4589         }
4590         i++;
4591       } else if (cmd.equals("-sleepBeforeRerun")) {
4592         if (i == args.length - 1) {
4593           errors.reportError(ERROR_CODE.WRONG_USAGE,
4594             "HBaseFsck: -sleepBeforeRerun needs a value.");
4595           return printUsageAndExit();
4596         }
4597         try {
4598           sleepBeforeRerun = Long.parseLong(args[i+1]);
4599         } catch (NumberFormatException e) {
4600           errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4601           return printUsageAndExit();
4602         }
4603         i++;
4604       } else if (cmd.equals("-sidelineDir")) {
4605         if (i == args.length - 1) {
4606           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4607           return printUsageAndExit();
4608         }
4609         i++;
4610         setSidelineDir(args[i]);
4611       } else if (cmd.equals("-fix")) {
4612         errors.reportError(ERROR_CODE.WRONG_USAGE,
4613           "This option is deprecated, please use  -fixAssignments instead.");
4614         setFixAssignments(true);
4615       } else if (cmd.equals("-fixAssignments")) {
4616         setFixAssignments(true);
4617       } else if (cmd.equals("-fixMeta")) {
4618         setFixMeta(true);
4619       } else if (cmd.equals("-noHdfsChecking")) {
4620         setCheckHdfs(false);
4621       } else if (cmd.equals("-fixHdfsHoles")) {
4622         setFixHdfsHoles(true);
4623       } else if (cmd.equals("-fixHdfsOrphans")) {
4624         setFixHdfsOrphans(true);
4625       } else if (cmd.equals("-fixTableOrphans")) {
4626         setFixTableOrphans(true);
4627       } else if (cmd.equals("-fixHdfsOverlaps")) {
4628         setFixHdfsOverlaps(true);
4629       } else if (cmd.equals("-fixVersionFile")) {
4630         setFixVersionFile(true);
4631       } else if (cmd.equals("-sidelineBigOverlaps")) {
4632         setSidelineBigOverlaps(true);
4633       } else if (cmd.equals("-fixSplitParents")) {
4634         setFixSplitParents(true);
4635       } else if (cmd.equals("-ignorePreCheckPermission")) {
4636         setIgnorePreCheckPermission(true);
4637       } else if (cmd.equals("-checkCorruptHFiles")) {
4638         checkCorruptHFiles = true;
4639       } else if (cmd.equals("-sidelineCorruptHFiles")) {
4640         sidelineCorruptHFiles = true;
4641       } else if (cmd.equals("-fixReferenceFiles")) {
4642         setFixReferenceFiles(true);
4643       } else if (cmd.equals("-fixEmptyMetaCells")) {
4644         setFixEmptyMetaCells(true);
4645       } else if (cmd.equals("-repair")) {
4646         // this attempts to merge overlapping hdfs regions, needs testing
4647         // under load
4648         setFixHdfsHoles(true);
4649         setFixHdfsOrphans(true);
4650         setFixMeta(true);
4651         setFixAssignments(true);
4652         setFixHdfsOverlaps(true);
4653         setFixVersionFile(true);
4654         setSidelineBigOverlaps(true);
4655         setFixSplitParents(false);
4656         setCheckHdfs(true);
4657         setFixReferenceFiles(true);
4658         setFixTableLocks(true);
4659       } else if (cmd.equals("-repairHoles")) {
4660         // this will make all missing hdfs regions available but may lose data
4661         setFixHdfsHoles(true);
4662         setFixHdfsOrphans(false);
4663         setFixMeta(true);
4664         setFixAssignments(true);
4665         setFixHdfsOverlaps(false);
4666         setSidelineBigOverlaps(false);
4667         setFixSplitParents(false);
4668         setCheckHdfs(true);
4669       } else if (cmd.equals("-maxOverlapsToSideline")) {
4670         if (i == args.length - 1) {
4671           errors.reportError(ERROR_CODE.WRONG_USAGE,
4672             "-maxOverlapsToSideline needs a numeric value argument.");
4673           return printUsageAndExit();
4674         }
4675         try {
4676           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4677           setMaxOverlapsToSideline(maxOverlapsToSideline);
4678         } catch (NumberFormatException e) {
4679           errors.reportError(ERROR_CODE.WRONG_USAGE,
4680             "-maxOverlapsToSideline needs a numeric value argument.");
4681           return printUsageAndExit();
4682         }
4683         i++;
4684       } else if (cmd.equals("-maxMerge")) {
4685         if (i == args.length - 1) {
4686           errors.reportError(ERROR_CODE.WRONG_USAGE,
4687             "-maxMerge needs a numeric value argument.");
4688           return printUsageAndExit();
4689         }
4690         try {
4691           int maxMerge = Integer.parseInt(args[i+1]);
4692           setMaxMerge(maxMerge);
4693         } catch (NumberFormatException e) {
4694           errors.reportError(ERROR_CODE.WRONG_USAGE,
4695             "-maxMerge needs a numeric value argument.");
4696           return printUsageAndExit();
4697         }
4698         i++;
4699       } else if (cmd.equals("-summary")) {
4700         setSummary();
4701       } else if (cmd.equals("-metaonly")) {
4702         setCheckMetaOnly();
4703       } else if (cmd.equals("-boundaries")) {
4704         setRegionBoundariesCheck();
4705       } else if (cmd.equals("-fixTableLocks")) {
4706         setFixTableLocks(true);
4707       } else if (cmd.equals("-fixReplication")) {
4708         setFixReplication(true);
4709       } else if (cmd.startsWith("-")) {
4710         errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4711         return printUsageAndExit();
4712       } else {
4713         includeTable(TableName.valueOf(cmd));
4714         errors.print("Allow checking/fixes for table: " + cmd);
4715       }
4716     }
4717 
4718     errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
4719 
4720     // pre-check current user has FS write permission or not
4721     try {
4722       preCheckPermission();
4723     } catch (AccessDeniedException ace) {
4724       Runtime.getRuntime().exit(-1);
4725     } catch (IOException ioe) {
4726       Runtime.getRuntime().exit(-1);
4727     }
4728 
4729     // do the real work of hbck
4730     connect();
4731 
4732     try {
4733       // if corrupt file mode is on, first fix them since they may be opened later
4734       if (checkCorruptHFiles || sidelineCorruptHFiles) {
4735         LOG.info("Checking all hfiles for corruption");
4736         HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4737         setHFileCorruptionChecker(hfcc); // so we can get result
4738         Collection<TableName> tables = getIncludedTables();
4739         Collection<Path> tableDirs = new ArrayList<Path>();
4740         Path rootdir = FSUtils.getRootDir(getConf());
4741         if (tables.size() > 0) {
4742           for (TableName t : tables) {
4743             tableDirs.add(FSUtils.getTableDir(rootdir, t));
4744           }
4745         } else {
4746           tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4747         }
4748         hfcc.checkTables(tableDirs);
4749         hfcc.report(errors);
4750       }
4751 
4752       // check and fix table integrity, region consistency.
4753       int code = onlineHbck();
4754       setRetCode(code);
4755       // If we have changed the HBase state it is better to run hbck again
4756       // to see if we haven't broken something else in the process.
4757       // We run it only once more because otherwise we can easily fall into
4758       // an infinite loop.
4759       if (shouldRerun()) {
4760         try {
4761           LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4762           Thread.sleep(sleepBeforeRerun);
4763         } catch (InterruptedException ie) {
4764           LOG.warn("Interrupted while sleeping");
4765           return this;
4766         }
4767         // Just report
4768         setFixAssignments(false);
4769         setFixMeta(false);
4770         setFixHdfsHoles(false);
4771         setFixHdfsOverlaps(false);
4772         setFixVersionFile(false);
4773         setFixTableOrphans(false);
4774         errors.resetErrors();
4775         code = onlineHbck();
4776         setRetCode(code);
4777       }
4778     } finally {
4779       IOUtils.closeQuietly(this);
4780     }
4781     return this;
4782   }
4783 
4784   /**
4785    * ls -r for debugging purposes
4786    */
4787   void debugLsr(Path p) throws IOException {
4788     debugLsr(getConf(), p, errors);
4789   }
4790 
4791   /**
4792    * ls -r for debugging purposes
4793    */
4794   public static void debugLsr(Configuration conf,
4795       Path p) throws IOException {
4796     debugLsr(conf, p, new PrintingErrorReporter());
4797   }
4798 
4799   /**
4800    * ls -r for debugging purposes
4801    */
4802   public static void debugLsr(Configuration conf,
4803       Path p, ErrorReporter errors) throws IOException {
4804     if (!LOG.isDebugEnabled() || p == null) {
4805       return;
4806     }
4807     FileSystem fs = p.getFileSystem(conf);
4808 
4809     if (!fs.exists(p)) {
4810       // nothing
4811       return;
4812     }
4813     errors.print(p.toString());
4814 
4815     if (fs.isFile(p)) {
4816       return;
4817     }
4818 
4819     if (fs.getFileStatus(p).isDirectory()) {
4820       FileStatus[] fss= fs.listStatus(p);
4821       for (FileStatus status : fss) {
4822         debugLsr(conf, status.getPath(), errors);
4823       }
4824     }
4825   }
4826 }