View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.Closeable;
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.io.InterruptedIOException;
24  import java.io.PrintWriter;
25  import java.io.StringWriter;
26  import java.net.InetAddress;
27  import java.net.URI;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collection;
31  import java.util.Collections;
32  import java.util.Comparator;
33  import java.util.HashMap;
34  import java.util.HashSet;
35  import java.util.Iterator;
36  import java.util.List;
37  import java.util.Map;
38  import java.util.Map.Entry;
39  import java.util.Set;
40  import java.util.SortedMap;
41  import java.util.SortedSet;
42  import java.util.TreeMap;
43  import java.util.TreeSet;
44  import java.util.concurrent.Callable;
45  import java.util.concurrent.ConcurrentSkipListMap;
46  import java.util.concurrent.ExecutionException;
47  import java.util.concurrent.ExecutorService;
48  import java.util.concurrent.Executors;
49  import java.util.concurrent.Future;
50  import java.util.concurrent.FutureTask;
51  import java.util.concurrent.ScheduledThreadPoolExecutor;
52  import java.util.concurrent.TimeUnit;
53  import java.util.concurrent.TimeoutException;
54  import java.util.concurrent.atomic.AtomicBoolean;
55  import java.util.concurrent.atomic.AtomicInteger;
56  
57  import org.apache.commons.io.IOUtils;
58  import org.apache.commons.lang.StringUtils;
59  import org.apache.commons.logging.Log;
60  import org.apache.commons.logging.LogFactory;
61  import org.apache.hadoop.conf.Configuration;
62  import org.apache.hadoop.conf.Configured;
63  import org.apache.hadoop.fs.FSDataOutputStream;
64  import org.apache.hadoop.fs.FileStatus;
65  import org.apache.hadoop.fs.FileSystem;
66  import org.apache.hadoop.fs.Path;
67  import org.apache.hadoop.fs.permission.FsAction;
68  import org.apache.hadoop.fs.permission.FsPermission;
69  import org.apache.hadoop.hbase.Abortable;
70  import org.apache.hadoop.hbase.Cell;
71  import org.apache.hadoop.hbase.ClusterStatus;
72  import org.apache.hadoop.hbase.CoordinatedStateException;
73  import org.apache.hadoop.hbase.HBaseConfiguration;
74  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
75  import org.apache.hadoop.hbase.HColumnDescriptor;
76  import org.apache.hadoop.hbase.HConstants;
77  import org.apache.hadoop.hbase.HRegionInfo;
78  import org.apache.hadoop.hbase.HRegionLocation;
79  import org.apache.hadoop.hbase.HTableDescriptor;
80  import org.apache.hadoop.hbase.KeyValue;
81  import org.apache.hadoop.hbase.MasterNotRunningException;
82  import org.apache.hadoop.hbase.MetaTableAccessor;
83  import org.apache.hadoop.hbase.RegionLocations;
84  import org.apache.hadoop.hbase.ServerName;
85  import org.apache.hadoop.hbase.TableName;
86  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
87  import org.apache.hadoop.hbase.classification.InterfaceAudience;
88  import org.apache.hadoop.hbase.classification.InterfaceStability;
89  import org.apache.hadoop.hbase.client.Admin;
90  import org.apache.hadoop.hbase.client.ClusterConnection;
91  import org.apache.hadoop.hbase.client.ConnectionFactory;
92  import org.apache.hadoop.hbase.client.Delete;
93  import org.apache.hadoop.hbase.client.Get;
94  import org.apache.hadoop.hbase.client.HBaseAdmin;
95  import org.apache.hadoop.hbase.client.HConnectable;
96  import org.apache.hadoop.hbase.client.HConnection;
97  import org.apache.hadoop.hbase.client.HConnectionManager;
98  import org.apache.hadoop.hbase.client.MetaScanner;
99  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
100 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
101 import org.apache.hadoop.hbase.client.Put;
102 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
103 import org.apache.hadoop.hbase.client.Result;
104 import org.apache.hadoop.hbase.client.RowMutations;
105 import org.apache.hadoop.hbase.client.Table;
106 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
107 import org.apache.hadoop.hbase.io.hfile.HFile;
108 import org.apache.hadoop.hbase.master.MasterFileSystem;
109 import org.apache.hadoop.hbase.master.RegionState;
110 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
111 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
112 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
113 import org.apache.hadoop.hbase.regionserver.HRegion;
114 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
115 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
116 import org.apache.hadoop.hbase.security.AccessDeniedException;
117 import org.apache.hadoop.hbase.security.UserProvider;
118 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
119 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
120 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
121 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
122 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
123 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
124 import org.apache.hadoop.hbase.wal.WALSplitter;
125 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
126 import org.apache.hadoop.hbase.zookeeper.ZKTableStateClientSideReader;
127 import org.apache.hadoop.hbase.zookeeper.ZKTableStateManager;
128 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
129 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
130 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
131 import org.apache.hadoop.ipc.RemoteException;
132 import org.apache.hadoop.security.UserGroupInformation;
133 import org.apache.hadoop.util.ReflectionUtils;
134 import org.apache.hadoop.util.Tool;
135 import org.apache.hadoop.util.ToolRunner;
136 import org.apache.zookeeper.KeeperException;
137 
138 import com.google.common.base.Joiner;
139 import com.google.common.base.Preconditions;
140 import com.google.common.collect.ImmutableList;
141 import com.google.common.collect.Lists;
142 import com.google.common.collect.Multimap;
143 import com.google.common.collect.Ordering;
144 import com.google.common.collect.TreeMultimap;
145 import com.google.protobuf.ServiceException;
146 
147 /**
148  * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
149  * table integrity problems in a corrupted HBase.
150  * <p>
151  * Region consistency checks verify that hbase:meta, region deployment on region
152  * servers and the state of data in HDFS (.regioninfo files) all are in
153  * accordance.
154  * <p>
155  * Table integrity checks verify that all possible row keys resolve to exactly
156  * one region of a table.  This means there are no individual degenerate
157  * or backwards regions; no holes between regions; and that there are no
158  * overlapping regions.
159  * <p>
160  * The general repair strategy works in two phases:
161  * <ol>
162  * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
163  * <li> Repair Region Consistency with hbase:meta and assignments
164  * </ol>
165  * <p>
166  * For table integrity repairs, the tables' region directories are scanned
167  * for .regioninfo files.  Each table's integrity is then verified.  If there
168  * are any orphan regions (regions with no .regioninfo files) or holes, new
169  * regions are fabricated.  Backwards regions are sidelined as well as empty
170  * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
171  * a new region is created and all data is merged into the new region.
172  * <p>
173  * Table integrity repairs deal solely with HDFS and could potentially be done
174  * offline -- the hbase region servers or master do not need to be running.
175  * This phase can eventually be used to completely reconstruct the hbase:meta table in
176  * an offline fashion.
177  * <p>
178  * Region consistency requires three conditions -- 1) valid .regioninfo file
179  * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
180  * and 3) a region is deployed only at the regionserver that was assigned to
181  * with proper state in the master.
182  * <p>
183  * Region consistency repairs require hbase to be online so that hbck can
184  * contact the HBase master and region servers.  The hbck#connect() method must
185  * first be called successfully.  Much of the region consistency information
186  * is transient and less risky to repair.
187  * <p>
188  * If hbck is run from the command line, there are a handful of arguments that
189  * can be used to limit the kinds of repairs hbck will do.  See the code in
190  * {@link #printUsageAndExit()} for more details.
191  */
192 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
193 @InterfaceStability.Evolving
194 public class HBaseFsck extends Configured implements Closeable {
195   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
196   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
197   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
198   private static boolean rsSupportsOffline = true;
199   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
200   private static final int DEFAULT_MAX_MERGE = 5;
201   private static final String TO_BE_LOADED = "to_be_loaded";
202   private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
203   private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5;
204   private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds
205   private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds
206   // We have to set the timeout value > HdfsConstants.LEASE_SOFTLIMIT_PERIOD.
207   // In HADOOP-2.6 and later, the Namenode proxy now created with custom RetryPolicy for
208   // AlreadyBeingCreatedException which is implies timeout on this operations up to
209   // HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds).
210   private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds
211 
212   /**********************
213    * Internal resources
214    **********************/
215   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
216   private ClusterStatus status;
217   private ClusterConnection connection;
218   private Admin admin;
219   private Table meta;
220   // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
221   protected ExecutorService executor;
222   private long startMillis = EnvironmentEdgeManager.currentTime();
223   private HFileCorruptionChecker hfcc;
224   private int retcode = 0;
225   private Path HBCK_LOCK_PATH;
226   private FSDataOutputStream hbckOutFd;
227   // This lock is to prevent cleanup of balancer resources twice between
228   // ShutdownHook and the main code. We cleanup only if the connect() is
229   // successful
230   private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
231 
232   /***********
233    * Options
234    ***********/
235   private static boolean details = false; // do we display the full report
236   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
237   private static boolean forceExclusive = false; // only this hbck can modify HBase
238   private static boolean disableBalancer = false; // disable load balancer to keep regions stable
239   private boolean fixAssignments = false; // fix assignment errors?
240   private boolean fixMeta = false; // fix meta errors?
241   private boolean checkHdfs = true; // load and check fs consistency?
242   private boolean fixHdfsHoles = false; // fix fs holes?
243   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
244   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
245   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
246   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
247   private boolean fixSplitParents = false; // fix lingering split parents
248   private boolean fixReferenceFiles = false; // fix lingering reference store file
249   private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
250   private boolean fixTableLocks = false; // fix table locks which are expired
251   private boolean fixTableZNodes = false; // fix table Znodes which are orphaned
252   private boolean fixAny = false; // Set to true if any of the fix is required.
253 
254   // limit checking/fixes to listed tables, if empty attempt to check/fix all
255   // hbase:meta are always checked
256   private Set<TableName> tablesIncluded = new HashSet<TableName>();
257   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
258   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
259   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
260   private Path sidelineDir = null;
261 
262   private boolean rerun = false; // if we tried to fix something, rerun hbck
263   private static boolean SUMMARY = false; // if we want to print less output
264   private boolean checkMetaOnly = false;
265   private boolean checkRegionBoundaries = false;
266   private boolean ignorePreCheckPermission = false; // if pre-check permission
267 
268   /*********
269    * State
270    *********/
271   final private ErrorReporter errors;
272   int fixes = 0;
273 
274   /**
275    * This map contains the state of all hbck items.  It maps from encoded region
276    * name to HbckInfo structure.  The information contained in HbckInfo is used
277    * to detect and correct consistency (hdfs/meta/deployment) problems.
278    */
279   private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
280   private TreeSet<TableName> disabledTables =
281     new TreeSet<TableName>();
282   // Empty regioninfo qualifiers in hbase:meta
283   private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
284 
285   /**
286    * This map from Tablename -> TableInfo contains the structures necessary to
287    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
288    * to prevent dupes.
289    *
290    * If tablesIncluded is empty, this map contains all tables.
291    * Otherwise, it contains only meta tables and tables in tablesIncluded,
292    * unless checkMetaOnly is specified, in which case, it contains only
293    * the meta table
294    */
295   private SortedMap<TableName, TableInfo> tablesInfo =
296       new ConcurrentSkipListMap<TableName, TableInfo>();
297 
298   /**
299    * When initially looking at HDFS, we attempt to find any orphaned data.
300    */
301   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
302 
303   private Map<TableName, Set<String>> orphanTableDirs =
304       new HashMap<TableName, Set<String>>();
305 
306   private Map<TableName, Set<String>> skippedRegions = new HashMap<TableName, Set<String>>();
307 
308   /**
309    * List of orphaned table ZNodes
310    */
311   private Set<TableName> orphanedTableZNodes = new HashSet<TableName>();
312   private final RetryCounterFactory lockFileRetryCounterFactory;
313   
314 
315   /**
316    * Constructor
317    *
318    * @param conf Configuration object
319    * @throws MasterNotRunningException if the master is not running
320    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
321    */
322   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
323       ZooKeeperConnectionException, IOException, ClassNotFoundException {
324     this(conf, createThreadPool(conf));
325   }
326 
327   private static ExecutorService createThreadPool(Configuration conf) {
328     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
329     return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
330   }
331 
332   /**
333    * Constructor
334    *
335    * @param conf
336    *          Configuration object
337    * @throws MasterNotRunningException
338    *           if the master is not running
339    * @throws ZooKeeperConnectionException
340    *           if unable to connect to ZooKeeper
341    */
342   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
343       ZooKeeperConnectionException, IOException, ClassNotFoundException {
344     super(conf);
345     errors = getErrorReporter(getConf());
346     this.executor = exec;
347     lockFileRetryCounterFactory = new RetryCounterFactory(
348       getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
349       getConf().getInt(
350         "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),
351       getConf().getInt(
352         "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));
353   }
354 
355   private class FileLockCallable implements Callable<FSDataOutputStream> {
356     RetryCounter retryCounter;
357 
358     public FileLockCallable(RetryCounter retryCounter) {
359       this.retryCounter = retryCounter;
360     }
361     @Override
362     public FSDataOutputStream call() throws IOException {
363       try {
364         FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
365         FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
366             HConstants.DATA_FILE_UMASK_KEY);
367         Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
368         fs.mkdirs(tmpDir);
369         HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
370         final FSDataOutputStream out = createFileWithRetries(fs, HBCK_LOCK_PATH, defaultPerms);
371         out.writeBytes(InetAddress.getLocalHost().toString());
372         out.flush();
373         return out;
374       } catch(RemoteException e) {
375         if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
376           return null;
377         } else {
378           throw e;
379         }
380       }
381     }
382 
383     private FSDataOutputStream createFileWithRetries(final FileSystem fs,
384         final Path hbckLockFilePath, final FsPermission defaultPerms)
385         throws IOException {
386 
387       IOException exception = null;
388       do {
389         try {
390           return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false);
391         } catch (IOException ioe) {
392           LOG.info("Failed to create lock file " + hbckLockFilePath.getName()
393               + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of "
394               + retryCounter.getMaxAttempts());
395           LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(), 
396               ioe);
397           try {
398             exception = ioe;
399             retryCounter.sleepUntilNextRetry();
400           } catch (InterruptedException ie) {
401             throw (InterruptedIOException) new InterruptedIOException(
402                 "Can't create lock file " + hbckLockFilePath.getName())
403             .initCause(ie);
404           }
405         }
406       } while (retryCounter.shouldRetry());
407 
408       throw exception;
409     }
410   }
411 
412   /**
413    * This method maintains a lock using a file. If the creation fails we return null
414    *
415    * @return FSDataOutputStream object corresponding to the newly opened lock file
416    * @throws IOException
417    */
418   private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
419     RetryCounter retryCounter = lockFileRetryCounterFactory.create();
420     FileLockCallable callable = new FileLockCallable(retryCounter);
421     ExecutorService executor = Executors.newFixedThreadPool(1);
422     FutureTask<FSDataOutputStream> futureTask = new FutureTask<FSDataOutputStream>(callable);
423     executor.execute(futureTask);
424     final int timeoutInSeconds = getConf().getInt(
425       "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT);
426     FSDataOutputStream stream = null;
427     try {
428       stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);
429     } catch (ExecutionException ee) {
430       LOG.warn("Encountered exception when opening lock file", ee);
431     } catch (InterruptedException ie) {
432       LOG.warn("Interrupted when opening lock file", ie);
433       Thread.currentThread().interrupt();
434     } catch (TimeoutException exception) {
435       // took too long to obtain lock
436       LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");
437       futureTask.cancel(true);
438     } finally {
439       executor.shutdownNow();
440     }
441     return stream;
442   }
443 
444   private void unlockHbck() {
445     if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) {
446       RetryCounter retryCounter = lockFileRetryCounterFactory.create();
447       do {
448         try {
449           IOUtils.closeQuietly(hbckOutFd);
450           FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()),
451               HBCK_LOCK_PATH, true);
452           LOG.info("Finishing hbck");
453           return;
454         } catch (IOException ioe) {
455           LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try="
456               + (retryCounter.getAttemptTimes() + 1) + " of "
457               + retryCounter.getMaxAttempts());
458           LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe);
459           try {
460             retryCounter.sleepUntilNextRetry();
461           } catch (InterruptedException ie) {
462             Thread.currentThread().interrupt();
463             LOG.warn("Interrupted while deleting lock file" +
464                 HBCK_LOCK_PATH);
465             return;
466           }
467         }
468       } while (retryCounter.shouldRetry());
469     }
470   }
471 
472   /**
473    * To repair region consistency, one must call connect() in order to repair
474    * online state.
475    */
476   public void connect() throws IOException {
477 
478     if (isExclusive()) {
479       // Grab the lock
480       hbckOutFd = checkAndMarkRunningHbck();
481       if (hbckOutFd == null) {
482         setRetCode(-1);
483         LOG.error("Another instance of hbck is fixing HBase, exiting this instance. " +
484             "[If you are sure no other instance is running, delete the lock file " +
485             HBCK_LOCK_PATH + " and rerun the tool]");
486         throw new IOException("Duplicate hbck - Abort");
487       }
488 
489       // Make sure to cleanup the lock
490       hbckLockCleanup.set(true);
491     }
492 
493 
494     // Add a shutdown hook to this thread, in case user tries to
495     // kill the hbck with a ctrl-c, we want to cleanup the lock so that
496     // it is available for further calls
497     Runtime.getRuntime().addShutdownHook(new Thread() {
498       @Override
499       public void run() {
500         IOUtils.closeQuietly(HBaseFsck.this);
501         unlockHbck();
502       }
503     });
504 
505     LOG.info("Launching hbck");
506 
507     connection = (ClusterConnection)ConnectionFactory.createConnection(getConf());
508     admin = connection.getAdmin();
509     meta = connection.getTable(TableName.META_TABLE_NAME);
510     status = admin.getClusterStatus();
511   }
512 
513   /**
514    * Get deployed regions according to the region servers.
515    */
516   private void loadDeployedRegions() throws IOException, InterruptedException {
517     // From the master, get a list of all known live region servers
518     Collection<ServerName> regionServers = status.getServers();
519     errors.print("Number of live region servers: " + regionServers.size());
520     if (details) {
521       for (ServerName rsinfo: regionServers) {
522         errors.print("  " + rsinfo.getServerName());
523       }
524     }
525 
526     // From the master, get a list of all dead region servers
527     Collection<ServerName> deadRegionServers = status.getDeadServerNames();
528     errors.print("Number of dead region servers: " + deadRegionServers.size());
529     if (details) {
530       for (ServerName name: deadRegionServers) {
531         errors.print("  " + name);
532       }
533     }
534 
535     // Print the current master name and state
536     errors.print("Master: " + status.getMaster());
537 
538     // Print the list of all backup masters
539     Collection<ServerName> backupMasters = status.getBackupMasters();
540     errors.print("Number of backup masters: " + backupMasters.size());
541     if (details) {
542       for (ServerName name: backupMasters) {
543         errors.print("  " + name);
544       }
545     }
546 
547     errors.print("Average load: " + status.getAverageLoad());
548     errors.print("Number of requests: " + status.getRequestsCount());
549     errors.print("Number of regions: " + status.getRegionsCount());
550 
551     Map<String, RegionState> rits = status.getRegionsInTransition();
552     errors.print("Number of regions in transition: " + rits.size());
553     if (details) {
554       for (RegionState state: rits.values()) {
555         errors.print("  " + state.toDescriptiveString());
556       }
557     }
558 
559     // Determine what's deployed
560     processRegionServers(regionServers);
561   }
562 
563   /**
564    * Clear the current state of hbck.
565    */
566   private void clearState() {
567     // Make sure regionInfo is empty before starting
568     fixes = 0;
569     regionInfoMap.clear();
570     emptyRegionInfoQualifiers.clear();
571     disabledTables.clear();
572     errors.clear();
573     tablesInfo.clear();
574     orphanHdfsDirs.clear();
575     skippedRegions.clear();
576   }
577 
578   /**
579    * This repair method analyzes hbase data in hdfs and repairs it to satisfy
580    * the table integrity rules.  HBase doesn't need to be online for this
581    * operation to work.
582    */
583   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
584     // Initial pass to fix orphans.
585     if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
586         || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
587       LOG.info("Loading regioninfos HDFS");
588       // if nothing is happening this should always complete in two iterations.
589       int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
590       int curIter = 0;
591       do {
592         clearState(); // clears hbck state and reset fixes to 0 and.
593         // repair what's on HDFS
594         restoreHdfsIntegrity();
595         curIter++;// limit the number of iterations.
596       } while (fixes > 0 && curIter <= maxIterations);
597 
598       // Repairs should be done in the first iteration and verification in the second.
599       // If there are more than 2 passes, something funny has happened.
600       if (curIter > 2) {
601         if (curIter == maxIterations) {
602           LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
603               + "Tables integrity may not be fully repaired!");
604         } else {
605           LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
606         }
607       }
608     }
609   }
610 
611   /**
612    * This repair method requires the cluster to be online since it contacts
613    * region servers and the masters.  It makes each region's state in HDFS, in
614    * hbase:meta, and deployments consistent.
615    *
616    * @return If &gt; 0 , number of errors detected, if &lt; 0 there was an unrecoverable
617    * error.  If 0, we have a clean hbase.
618    */
619   public int onlineConsistencyRepair() throws IOException, KeeperException,
620     InterruptedException {
621     clearState();
622 
623     // get regions according to what is online on each RegionServer
624     loadDeployedRegions();
625     // check whether hbase:meta is deployed and online
626     recordMetaRegion();
627     // Check if hbase:meta is found only once and in the right place
628     if (!checkMetaRegion()) {
629       String errorMsg = "hbase:meta table is not consistent. ";
630       if (shouldFixAssignments()) {
631         errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
632       } else {
633         errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
634       }
635       errors.reportError(errorMsg + " Exiting...");
636       return -2;
637     }
638     // Not going with further consistency check for tables when hbase:meta itself is not consistent.
639     LOG.info("Loading regionsinfo from the hbase:meta table");
640     boolean success = loadMetaEntries();
641     if (!success) return -1;
642 
643     // Empty cells in hbase:meta?
644     reportEmptyMetaCells();
645 
646     // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
647     if (shouldFixEmptyMetaCells()) {
648       fixEmptyMetaCells();
649     }
650 
651     // get a list of all tables that have not changed recently.
652     if (!checkMetaOnly) {
653       reportTablesInFlux();
654     }
655 
656     // load regiondirs and regioninfos from HDFS
657     if (shouldCheckHdfs()) {
658       LOG.info("Loading region directories from HDFS");
659       loadHdfsRegionDirs();
660       LOG.info("Loading region information from HDFS");
661       loadHdfsRegionInfos();
662     }
663 
664     // Get disabled tables from ZooKeeper
665     loadDisabledTables();
666 
667     // fix the orphan tables
668     fixOrphanTables();
669 
670     LOG.info("Checking and fixing region consistency");
671     // Check and fix consistency
672     checkAndFixConsistency();
673 
674     // Check integrity (does not fix)
675     checkIntegrity();
676     return errors.getErrorList().size();
677   }
678 
679   /**
680    * Contacts the master and prints out cluster-wide information
681    * @return 0 on success, non-zero on failure
682    */
683   public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
684     // print hbase server version
685     errors.print("Version: " + status.getHBaseVersion());
686     offlineHdfsIntegrityRepair();
687 
688     boolean oldBalancer = false;
689     if (shouldDisableBalancer()) {
690       oldBalancer = admin.setBalancerRunning(false, true);
691     }
692 
693     try {
694       onlineConsistencyRepair();
695     }
696     finally {
697       // Only restore the balancer if it was true when we started repairing and
698       // we actually disabled it. Otherwise, we might clobber another run of
699       // hbck that has just restored it.
700       if (shouldDisableBalancer() && oldBalancer) {
701         admin.setBalancerRunning(oldBalancer, false);
702       }
703     }
704 
705     if (checkRegionBoundaries) {
706       checkRegionBoundaries();
707     }
708 
709     offlineReferenceFileRepair();
710 
711     checkAndFixTableLocks();
712 
713     // Check (and fix if requested) orphaned table ZNodes
714     checkAndFixOrphanedTableZNodes();
715 
716     // Remove the hbck lock
717     unlockHbck();
718 
719     // Print table summary
720     printTableSummary(tablesInfo);
721     return errors.summarize();
722   }
723 
724   public static byte[] keyOnly (byte[] b) {
725     if (b == null)
726       return b;
727     int rowlength = Bytes.toShort(b, 0);
728     byte[] result = new byte[rowlength];
729     System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
730     return result;
731   }
732 
733   @Override
734   public void close() throws IOException {
735     IOUtils.closeQuietly(admin);
736     IOUtils.closeQuietly(meta);
737     IOUtils.closeQuietly(connection);
738   }
739 
740   private static class RegionBoundariesInformation {
741     public byte [] regionName;
742     public byte [] metaFirstKey;
743     public byte [] metaLastKey;
744     public byte [] storesFirstKey;
745     public byte [] storesLastKey;
746     @Override
747     public String toString () {
748       return "regionName=" + Bytes.toStringBinary(regionName) +
749              "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
750              "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
751              "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
752              "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
753     }
754   }
755 
756   public void checkRegionBoundaries() {
757     try {
758       ByteArrayComparator comparator = new ByteArrayComparator();
759       List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), connection, false);
760       final RegionBoundariesInformation currentRegionBoundariesInformation =
761           new RegionBoundariesInformation();
762       Path hbaseRoot = FSUtils.getRootDir(getConf());
763       for (HRegionInfo regionInfo : regions) {
764         Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
765         currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
766         // For each region, get the start and stop key from the META and compare them to the
767         // same information from the Stores.
768         Path path = new Path(tableDir, regionInfo.getEncodedName());
769         FileSystem fs = path.getFileSystem(getConf());
770         FileStatus[] files = fs.listStatus(path);
771         // For all the column families in this region...
772         byte[] storeFirstKey = null;
773         byte[] storeLastKey = null;
774         for (FileStatus file : files) {
775           String fileName = file.getPath().toString();
776           fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
777           if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
778             FileStatus[] storeFiles = fs.listStatus(file.getPath());
779             // For all the stores in this column family.
780             for (FileStatus storeFile : storeFiles) {
781               HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
782                   getConf()), getConf());
783               if ((reader.getFirstKey() != null)
784                   && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
785                       reader.getFirstKey()) > 0))) {
786                 storeFirstKey = reader.getFirstKey();
787               }
788               if ((reader.getLastKey() != null)
789                   && ((storeLastKey == null) || (comparator.compare(storeLastKey,
790                       reader.getLastKey())) < 0)) {
791                 storeLastKey = reader.getLastKey();
792               }
793               reader.close();
794             }
795           }
796         }
797         currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
798         currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
799         currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
800         currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
801         if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
802           currentRegionBoundariesInformation.metaFirstKey = null;
803         if (currentRegionBoundariesInformation.metaLastKey.length == 0)
804           currentRegionBoundariesInformation.metaLastKey = null;
805 
806         // For a region to be correct, we need the META start key to be smaller or equal to the
807         // smallest start key from all the stores, and the start key from the next META entry to
808         // be bigger than the last key from all the current stores. First region start key is null;
809         // Last region end key is null; some regions can be empty and not have any store.
810 
811         boolean valid = true;
812         // Checking start key.
813         if ((currentRegionBoundariesInformation.storesFirstKey != null)
814             && (currentRegionBoundariesInformation.metaFirstKey != null)) {
815           valid = valid
816               && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
817                 currentRegionBoundariesInformation.metaFirstKey) >= 0;
818         }
819         // Checking stop key.
820         if ((currentRegionBoundariesInformation.storesLastKey != null)
821             && (currentRegionBoundariesInformation.metaLastKey != null)) {
822           valid = valid
823               && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
824                 currentRegionBoundariesInformation.metaLastKey) < 0;
825         }
826         if (!valid) {
827           errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
828             tablesInfo.get(regionInfo.getTable()));
829           LOG.warn("Region's boundaries not alligned between stores and META for:");
830           LOG.warn(currentRegionBoundariesInformation);
831         }
832       }
833     } catch (IOException e) {
834       LOG.error(e);
835     }
836   }
837 
838   /**
839    * Iterates through the list of all orphan/invalid regiondirs.
840    */
841   private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
842     for (HbckInfo hi : orphanHdfsDirs) {
843       LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
844       adoptHdfsOrphan(hi);
845     }
846   }
847 
848   /**
849    * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
850    * these orphans by creating a new region, and moving the column families,
851    * recovered edits, WALs, into the new region dir.  We determine the region
852    * startkey and endkeys by looking at all of the hfiles inside the column
853    * families to identify the min and max keys. The resulting region will
854    * likely violate table integrity but will be dealt with by merging
855    * overlapping regions.
856    */
857   @SuppressWarnings("deprecation")
858   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
859     Path p = hi.getHdfsRegionDir();
860     FileSystem fs = p.getFileSystem(getConf());
861     FileStatus[] dirs = fs.listStatus(p);
862     if (dirs == null) {
863       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
864           p + ". This dir could probably be deleted.");
865       return ;
866     }
867 
868     TableName tableName = hi.getTableName();
869     TableInfo tableInfo = tablesInfo.get(tableName);
870     Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
871     HTableDescriptor template = tableInfo.getHTD();
872 
873     // find min and max key values
874     Pair<byte[],byte[]> orphanRegionRange = null;
875     for (FileStatus cf : dirs) {
876       String cfName= cf.getPath().getName();
877       // TODO Figure out what the special dirs are
878       if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
879 
880       FileStatus[] hfiles = fs.listStatus(cf.getPath());
881       for (FileStatus hfile : hfiles) {
882         byte[] start, end;
883         HFile.Reader hf = null;
884         try {
885           CacheConfig cacheConf = new CacheConfig(getConf());
886           hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
887           hf.loadFileInfo();
888           KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
889           start = startKv.getRow();
890           KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
891           end = endKv.getRow();
892         } catch (IOException ioe) {
893           LOG.warn("Problem reading orphan file " + hfile + ", skipping");
894           continue;
895         } catch (NullPointerException ioe) {
896           LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
897           continue;
898         } finally {
899           if (hf != null) {
900             hf.close();
901           }
902         }
903 
904         // expand the range to include the range of all hfiles
905         if (orphanRegionRange == null) {
906           // first range
907           orphanRegionRange = new Pair<byte[], byte[]>(start, end);
908         } else {
909           // TODO add test
910 
911           // expand range only if the hfile is wider.
912           if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
913             orphanRegionRange.setFirst(start);
914           }
915           if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
916             orphanRegionRange.setSecond(end);
917           }
918         }
919       }
920     }
921     if (orphanRegionRange == null) {
922       LOG.warn("No data in dir " + p + ", sidelining data");
923       fixes++;
924       sidelineRegionDir(fs, hi);
925       return;
926     }
927     LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
928         Bytes.toString(orphanRegionRange.getSecond()) + ")");
929 
930     // create new region on hdfs.  move data into place.
931     HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(), 
932       Bytes.add(orphanRegionRange.getSecond(), new byte[1]));
933     LOG.info("Creating new region : " + hri);
934     HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
935     Path target = region.getRegionFileSystem().getRegionDir();
936 
937     // rename all the data to new region
938     mergeRegionDirs(target, hi);
939     fixes++;
940   }
941 
942   /**
943    * This method determines if there are table integrity errors in HDFS.  If
944    * there are errors and the appropriate "fix" options are enabled, the method
945    * will first correct orphan regions making them into legit regiondirs, and
946    * then reload to merge potentially overlapping regions.
947    *
948    * @return number of table integrity errors found
949    */
950   private int restoreHdfsIntegrity() throws IOException, InterruptedException {
951     // Determine what's on HDFS
952     LOG.info("Loading HBase regioninfo from HDFS...");
953     loadHdfsRegionDirs(); // populating regioninfo table.
954 
955     int errs = errors.getErrorList().size();
956     // First time just get suggestions.
957     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
958     checkHdfsIntegrity(false, false);
959 
960     if (errors.getErrorList().size() == errs) {
961       LOG.info("No integrity errors.  We are done with this phase. Glorious.");
962       return 0;
963     }
964 
965     if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
966       adoptHdfsOrphans(orphanHdfsDirs);
967       // TODO optimize by incrementally adding instead of reloading.
968     }
969 
970     // Make sure there are no holes now.
971     if (shouldFixHdfsHoles()) {
972       clearState(); // this also resets # fixes.
973       loadHdfsRegionDirs();
974       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
975       tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
976     }
977 
978     // Now we fix overlaps
979     if (shouldFixHdfsOverlaps()) {
980       // second pass we fix overlaps.
981       clearState(); // this also resets # fixes.
982       loadHdfsRegionDirs();
983       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
984       tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
985     }
986 
987     return errors.getErrorList().size();
988   }
989 
990   /**
991    * Scan all the store file names to find any lingering reference files,
992    * which refer to some none-exiting files. If "fix" option is enabled,
993    * any lingering reference file will be sidelined if found.
994    * <p>
995    * Lingering reference file prevents a region from opening. It has to
996    * be fixed before a cluster can start properly.
997    */
998   private void offlineReferenceFileRepair() throws IOException {
999     Configuration conf = getConf();
1000     Path hbaseRoot = FSUtils.getRootDir(conf);
1001     FileSystem fs = hbaseRoot.getFileSystem(conf);
1002     LOG.info("Computing mapping of all store files");
1003     Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot, errors);
1004     errors.print("");
1005     LOG.info("Validating mapping using HDFS state");
1006     for (Path path: allFiles.values()) {
1007       boolean isReference = false;
1008       try {
1009         isReference = StoreFileInfo.isReference(path);
1010       } catch (Throwable t) {
1011         // Ignore. Some files may not be store files at all.
1012         // For example, files under .oldlogs folder in hbase:meta
1013         // Warning message is already logged by
1014         // StoreFile#isReference.
1015       }
1016       if (!isReference) continue;
1017 
1018       Path referredToFile = StoreFileInfo.getReferredToFile(path);
1019       if (fs.exists(referredToFile)) continue;  // good, expected
1020 
1021       // Found a lingering reference file
1022       errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
1023         "Found lingering reference file " + path);
1024       if (!shouldFixReferenceFiles()) continue;
1025 
1026       // Now, trying to fix it since requested
1027       boolean success = false;
1028       String pathStr = path.toString();
1029 
1030       // A reference file path should be like
1031       // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
1032       // Up 5 directories to get the root folder.
1033       // So the file will be sidelined to a similar folder structure.
1034       int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
1035       for (int i = 0; index > 0 && i < 5; i++) {
1036         index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
1037       }
1038       if (index > 0) {
1039         Path rootDir = getSidelineDir();
1040         Path dst = new Path(rootDir, pathStr.substring(index + 1));
1041         fs.mkdirs(dst.getParent());
1042         LOG.info("Trying to sildeline reference file "
1043           + path + " to " + dst);
1044         setShouldRerun();
1045 
1046         success = fs.rename(path, dst);
1047       }
1048       if (!success) {
1049         LOG.error("Failed to sideline reference file " + path);
1050       }
1051     }
1052   }
1053 
1054   /**
1055    * TODO -- need to add tests for this.
1056    */
1057   private void reportEmptyMetaCells() {
1058     errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
1059       emptyRegionInfoQualifiers.size());
1060     if (details) {
1061       for (Result r: emptyRegionInfoQualifiers) {
1062         errors.print("  " + r);
1063       }
1064     }
1065   }
1066 
1067   /**
1068    * TODO -- need to add tests for this.
1069    */
1070   private void reportTablesInFlux() {
1071     AtomicInteger numSkipped = new AtomicInteger(0);
1072     HTableDescriptor[] allTables = getTables(numSkipped);
1073     errors.print("Number of Tables: " + allTables.length);
1074     if (details) {
1075       if (numSkipped.get() > 0) {
1076         errors.detail("Number of Tables in flux: " + numSkipped.get());
1077       }
1078       for (HTableDescriptor td : allTables) {
1079         errors.detail("  Table: " + td.getTableName() + "\t" +
1080                            (td.isReadOnly() ? "ro" : "rw") + "\t" +
1081                             (td.isMetaRegion() ? "META" : "    ") + "\t" +
1082                            " families: " + td.getFamilies().size());
1083       }
1084     }
1085   }
1086 
1087   public ErrorReporter getErrors() {
1088     return errors;
1089   }
1090 
1091   /**
1092    * Read the .regioninfo file from the file system.  If there is no
1093    * .regioninfo, add it to the orphan hdfs region list.
1094    */
1095   private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
1096     Path regionDir = hbi.getHdfsRegionDir();
1097     if (regionDir == null) {
1098       if (hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1099         // Log warning only for default/ primary replica with no region dir
1100         LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
1101       }
1102       return;
1103     }
1104 
1105     if (hbi.hdfsEntry.hri != null) {
1106       // already loaded data
1107       return;
1108     }
1109 
1110     FileSystem fs = FileSystem.get(getConf());
1111     HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
1112     LOG.debug("HRegionInfo read: " + hri.toString());
1113     hbi.hdfsEntry.hri = hri;
1114   }
1115 
1116   /**
1117    * Exception thrown when a integrity repair operation fails in an
1118    * unresolvable way.
1119    */
1120   public static class RegionRepairException extends IOException {
1121     private static final long serialVersionUID = 1L;
1122     final IOException ioe;
1123     public RegionRepairException(String s, IOException ioe) {
1124       super(s);
1125       this.ioe = ioe;
1126     }
1127   }
1128 
1129   /**
1130    * Populate hbi's from regionInfos loaded from file system.
1131    */
1132   private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
1133       throws IOException, InterruptedException {
1134     tablesInfo.clear(); // regenerating the data
1135     // generate region split structure
1136     Collection<HbckInfo> hbckInfos = regionInfoMap.values();
1137 
1138     // Parallelized read of .regioninfo files.
1139     List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
1140     List<Future<Void>> hbiFutures;
1141 
1142     for (HbckInfo hbi : hbckInfos) {
1143       WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1144       hbis.add(work);
1145     }
1146 
1147     // Submit and wait for completion
1148     hbiFutures = executor.invokeAll(hbis);
1149 
1150     for(int i=0; i<hbiFutures.size(); i++) {
1151       WorkItemHdfsRegionInfo work = hbis.get(i);
1152       Future<Void> f = hbiFutures.get(i);
1153       try {
1154         f.get();
1155       } catch(ExecutionException e) {
1156         LOG.warn("Failed to read .regioninfo file for region " +
1157               work.hbi.getRegionNameAsString(), e.getCause());
1158       }
1159     }
1160 
1161     Path hbaseRoot = FSUtils.getRootDir(getConf());
1162     FileSystem fs = hbaseRoot.getFileSystem(getConf());
1163     // serialized table info gathering.
1164     for (HbckInfo hbi: hbckInfos) {
1165 
1166       if (hbi.getHdfsHRI() == null) {
1167         // was an orphan
1168         continue;
1169       }
1170 
1171 
1172       // get table name from hdfs, populate various HBaseFsck tables.
1173       TableName tableName = hbi.getTableName();
1174       if (tableName == null) {
1175         // There was an entry in hbase:meta not in the HDFS?
1176         LOG.warn("tableName was null for: " + hbi);
1177         continue;
1178       }
1179 
1180       TableInfo modTInfo = tablesInfo.get(tableName);
1181       if (modTInfo == null) {
1182         // only executed once per table.
1183         modTInfo = new TableInfo(tableName);
1184         tablesInfo.put(tableName, modTInfo);
1185         try {
1186           HTableDescriptor htd =
1187               FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1188           modTInfo.htds.add(htd);
1189         } catch (IOException ioe) {
1190           if (!orphanTableDirs.containsKey(tableName)) {
1191             LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1192             //should only report once for each table
1193             errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1194                 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1195             Set<String> columns = new HashSet<String>();
1196             orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1197           }
1198         }
1199       }
1200       if (!hbi.isSkipChecks()) {
1201         modTInfo.addRegionInfo(hbi);
1202       }
1203     }
1204 
1205     loadTableInfosForTablesWithNoRegion();
1206     errors.print("");
1207 
1208     return tablesInfo;
1209   }
1210 
1211   /**
1212    * To get the column family list according to the column family dirs
1213    * @param columns
1214    * @param hbi
1215    * @return a set of column families
1216    * @throws IOException
1217    */
1218   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
1219     Path regionDir = hbi.getHdfsRegionDir();
1220     FileSystem fs = regionDir.getFileSystem(getConf());
1221     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1222     for (FileStatus subdir : subDirs) {
1223       String columnfamily = subdir.getPath().getName();
1224       columns.add(columnfamily);
1225     }
1226     return columns;
1227   }
1228 
1229   /**
1230    * To fabricate a .tableinfo file with following contents<br>
1231    * 1. the correct tablename <br>
1232    * 2. the correct colfamily list<br>
1233    * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1234    * @throws IOException
1235    */
1236   private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1237       Set<String> columns) throws IOException {
1238     if (columns ==null || columns.isEmpty()) return false;
1239     HTableDescriptor htd = new HTableDescriptor(tableName);
1240     for (String columnfamimly : columns) {
1241       htd.addFamily(new HColumnDescriptor(columnfamimly));
1242     }
1243     fstd.createTableDescriptor(htd, true);
1244     return true;
1245   }
1246 
1247   /**
1248    * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1249    * @throws IOException
1250    */
1251   public void fixEmptyMetaCells() throws IOException {
1252     if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1253       LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1254       for (Result region : emptyRegionInfoQualifiers) {
1255         deleteMetaRegion(region.getRow());
1256         errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1257       }
1258       emptyRegionInfoQualifiers.clear();
1259     }
1260   }
1261 
1262   /**
1263    * To fix orphan table by creating a .tableinfo file under tableDir <br>
1264    * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1265    * 2. else create a default .tableinfo file with following items<br>
1266    * &nbsp;2.1 the correct tablename <br>
1267    * &nbsp;2.2 the correct colfamily list<br>
1268    * &nbsp;2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1269    * @throws IOException
1270    */
1271   public void fixOrphanTables() throws IOException {
1272     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1273 
1274       List<TableName> tmpList = new ArrayList<TableName>();
1275       tmpList.addAll(orphanTableDirs.keySet());
1276       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1277       Iterator<Entry<TableName, Set<String>>> iter =
1278           orphanTableDirs.entrySet().iterator();
1279       int j = 0;
1280       int numFailedCase = 0;
1281       FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1282       while (iter.hasNext()) {
1283         Entry<TableName, Set<String>> entry =
1284             iter.next();
1285         TableName tableName = entry.getKey();
1286         LOG.info("Trying to fix orphan table error: " + tableName);
1287         if (j < htds.length) {
1288           if (tableName.equals(htds[j].getTableName())) {
1289             HTableDescriptor htd = htds[j];
1290             LOG.info("fixing orphan table: " + tableName + " from cache");
1291             fstd.createTableDescriptor(htd, true);
1292             j++;
1293             iter.remove();
1294           }
1295         } else {
1296           if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1297             LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1298             LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1299             iter.remove();
1300           } else {
1301             LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1302             numFailedCase++;
1303           }
1304         }
1305         fixes++;
1306       }
1307 
1308       if (orphanTableDirs.isEmpty()) {
1309         // all orphanTableDirs are luckily recovered
1310         // re-run doFsck after recovering the .tableinfo file
1311         setShouldRerun();
1312         LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1313       } else if (numFailedCase > 0) {
1314         LOG.error("Failed to fix " + numFailedCase
1315             + " OrphanTables with default .tableinfo files");
1316       }
1317 
1318     }
1319     //cleanup the list
1320     orphanTableDirs.clear();
1321 
1322   }
1323 
1324   /**
1325    * This borrows code from MasterFileSystem.bootstrap()
1326    *
1327    * @return an open hbase:meta HRegion
1328    */
1329   private HRegion createNewMeta() throws IOException {
1330       Path rootdir = FSUtils.getRootDir(getConf());
1331     Configuration c = getConf();
1332     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1333     HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
1334     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false);
1335     HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor);
1336     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
1337     return meta;
1338   }
1339 
1340   /**
1341    * Generate set of puts to add to new meta.  This expects the tables to be
1342    * clean with no overlaps or holes.  If there are any problems it returns null.
1343    *
1344    * @return An array list of puts to do in bulk, null if tables have problems
1345    */
1346   private ArrayList<Put> generatePuts(
1347       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1348     ArrayList<Put> puts = new ArrayList<Put>();
1349     boolean hasProblems = false;
1350     for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1351       TableName name = e.getKey();
1352 
1353       // skip "hbase:meta"
1354       if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1355         continue;
1356       }
1357 
1358       TableInfo ti = e.getValue();
1359       for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1360           .entrySet()) {
1361         Collection<HbckInfo> his = spl.getValue();
1362         int sz = his.size();
1363         if (sz != 1) {
1364           // problem
1365           LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1366               + " had " +  sz + " regions instead of exactly 1." );
1367           hasProblems = true;
1368           continue;
1369         }
1370 
1371         // add the row directly to meta.
1372         HbckInfo hi = his.iterator().next();
1373         HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
1374         Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1375         puts.add(p);
1376       }
1377     }
1378     return hasProblems ? null : puts;
1379   }
1380 
1381   /**
1382    * Suggest fixes for each table
1383    */
1384   private void suggestFixes(
1385       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1386     logParallelMerge();
1387     for (TableInfo tInfo : tablesInfo.values()) {
1388       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1389       tInfo.checkRegionChain(handler);
1390     }
1391   }
1392 
1393   /**
1394    * Rebuilds meta from information in hdfs/fs.  Depends on configuration
1395    * settings passed into hbck constructor to point to a particular fs/dir.
1396    *
1397    * @param fix flag that determines if method should attempt to fix holes
1398    * @return true if successful, false if attempt failed.
1399    */
1400   public boolean rebuildMeta(boolean fix) throws IOException,
1401       InterruptedException {
1402 
1403     // TODO check to make sure hbase is offline. (or at least the table
1404     // currently being worked on is off line)
1405 
1406     // Determine what's on HDFS
1407     LOG.info("Loading HBase regioninfo from HDFS...");
1408     loadHdfsRegionDirs(); // populating regioninfo table.
1409 
1410     int errs = errors.getErrorList().size();
1411     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1412     checkHdfsIntegrity(false, false);
1413 
1414     // make sure ok.
1415     if (errors.getErrorList().size() != errs) {
1416       // While in error state, iterate until no more fixes possible
1417       while(true) {
1418         fixes = 0;
1419         suggestFixes(tablesInfo);
1420         errors.clear();
1421         loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1422         checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1423 
1424         int errCount = errors.getErrorList().size();
1425 
1426         if (fixes == 0) {
1427           if (errCount > 0) {
1428             return false; // failed to fix problems.
1429           } else {
1430             break; // no fixes and no problems? drop out and fix stuff!
1431           }
1432         }
1433       }
1434     }
1435 
1436     // we can rebuild, move old meta out of the way and start
1437     LOG.info("HDFS regioninfo's seems good.  Sidelining old hbase:meta");
1438     Path backupDir = sidelineOldMeta();
1439 
1440     LOG.info("Creating new hbase:meta");
1441     HRegion meta = createNewMeta();
1442 
1443     // populate meta
1444     List<Put> puts = generatePuts(tablesInfo);
1445     if (puts == null) {
1446       LOG.fatal("Problem encountered when creating new hbase:meta entries.  " +
1447         "You may need to restore the previously sidelined hbase:meta");
1448       return false;
1449     }
1450     meta.batchMutate(puts.toArray(new Put[puts.size()]));
1451     HRegion.closeHRegion(meta);
1452     LOG.info("Success! hbase:meta table rebuilt.");
1453     LOG.info("Old hbase:meta is moved into " + backupDir);
1454     return true;
1455   }
1456 
1457   /**
1458    * Log an appropriate message about whether or not overlapping merges are computed in parallel.
1459    */
1460   private void logParallelMerge() {
1461     if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
1462       LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
1463           " false to run serially.");
1464     } else {
1465       LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
1466           " true to run in parallel.");
1467     }
1468   }
1469 
1470   private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1471       boolean fixOverlaps) throws IOException {
1472     LOG.info("Checking HBase region split map from HDFS data...");
1473     logParallelMerge();
1474     for (TableInfo tInfo : tablesInfo.values()) {
1475       TableIntegrityErrorHandler handler;
1476       if (fixHoles || fixOverlaps) {
1477         handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1478           fixHoles, fixOverlaps);
1479       } else {
1480         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1481       }
1482       if (!tInfo.checkRegionChain(handler)) {
1483         // should dump info as well.
1484         errors.report("Found inconsistency in table " + tInfo.getName());
1485       }
1486     }
1487     return tablesInfo;
1488   }
1489 
1490   private Path getSidelineDir() throws IOException {
1491     if (sidelineDir == null) {
1492       Path hbaseDir = FSUtils.getRootDir(getConf());
1493       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1494       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1495           + startMillis);
1496     }
1497     return sidelineDir;
1498   }
1499 
1500   /**
1501    * Sideline a region dir (instead of deleting it)
1502    */
1503   Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1504     return sidelineRegionDir(fs, null, hi);
1505   }
1506 
1507   /**
1508    * Sideline a region dir (instead of deleting it)
1509    *
1510    * @param parentDir if specified, the region will be sidelined to
1511    * folder like .../parentDir/<table name>/<region name>. The purpose
1512    * is to group together similar regions sidelined, for example, those
1513    * regions should be bulk loaded back later on. If null, it is ignored.
1514    */
1515   Path sidelineRegionDir(FileSystem fs,
1516       String parentDir, HbckInfo hi) throws IOException {
1517     TableName tableName = hi.getTableName();
1518     Path regionDir = hi.getHdfsRegionDir();
1519 
1520     if (!fs.exists(regionDir)) {
1521       LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1522       return null;
1523     }
1524 
1525     Path rootDir = getSidelineDir();
1526     if (parentDir != null) {
1527       rootDir = new Path(rootDir, parentDir);
1528     }
1529     Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1530     Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1531     fs.mkdirs(sidelineRegionDir);
1532     boolean success = false;
1533     FileStatus[] cfs =  fs.listStatus(regionDir);
1534     if (cfs == null) {
1535       LOG.info("Region dir is empty: " + regionDir);
1536     } else {
1537       for (FileStatus cf : cfs) {
1538         Path src = cf.getPath();
1539         Path dst =  new Path(sidelineRegionDir, src.getName());
1540         if (fs.isFile(src)) {
1541           // simple file
1542           success = fs.rename(src, dst);
1543           if (!success) {
1544             String msg = "Unable to rename file " + src +  " to " + dst;
1545             LOG.error(msg);
1546             throw new IOException(msg);
1547           }
1548           continue;
1549         }
1550 
1551         // is a directory.
1552         fs.mkdirs(dst);
1553 
1554         LOG.info("Sidelining files from " + src + " into containing region " + dst);
1555         // FileSystem.rename is inconsistent with directories -- if the
1556         // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1557         // it moves the src into the dst dir resulting in (foo/a/b).  If
1558         // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1559         FileStatus[] hfiles = fs.listStatus(src);
1560         if (hfiles != null && hfiles.length > 0) {
1561           for (FileStatus hfile : hfiles) {
1562             success = fs.rename(hfile.getPath(), dst);
1563             if (!success) {
1564               String msg = "Unable to rename file " + src +  " to " + dst;
1565               LOG.error(msg);
1566               throw new IOException(msg);
1567             }
1568           }
1569         }
1570         LOG.debug("Sideline directory contents:");
1571         debugLsr(sidelineRegionDir);
1572       }
1573     }
1574 
1575     LOG.info("Removing old region dir: " + regionDir);
1576     success = fs.delete(regionDir, true);
1577     if (!success) {
1578       String msg = "Unable to delete dir " + regionDir;
1579       LOG.error(msg);
1580       throw new IOException(msg);
1581     }
1582     return sidelineRegionDir;
1583   }
1584 
1585   /**
1586    * Side line an entire table.
1587    */
1588   void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1589       Path backupHbaseDir) throws IOException {
1590     Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1591     if (fs.exists(tableDir)) {
1592       Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1593       fs.mkdirs(backupTableDir.getParent());
1594       boolean success = fs.rename(tableDir, backupTableDir);
1595       if (!success) {
1596         throw new IOException("Failed to move  " + tableName + " from "
1597             +  tableDir + " to " + backupTableDir);
1598       }
1599     } else {
1600       LOG.info("No previous " + tableName +  " exists.  Continuing.");
1601     }
1602   }
1603 
1604   /**
1605    * @return Path to backup of original directory
1606    */
1607   Path sidelineOldMeta() throws IOException {
1608     // put current hbase:meta aside.
1609     Path hbaseDir = FSUtils.getRootDir(getConf());
1610     FileSystem fs = hbaseDir.getFileSystem(getConf());
1611     Path backupDir = getSidelineDir();
1612     fs.mkdirs(backupDir);
1613 
1614     try {
1615       sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1616     } catch (IOException e) {
1617         LOG.fatal("... failed to sideline meta. Currently in inconsistent state.  To restore "
1618             + "try to rename hbase:meta in " + backupDir.getName() + " to "
1619             + hbaseDir.getName() + ".", e);
1620       throw e; // throw original exception
1621     }
1622     return backupDir;
1623   }
1624 
1625   /**
1626    * Load the list of disabled tables in ZK into local set.
1627    * @throws ZooKeeperConnectionException
1628    * @throws IOException
1629    */
1630   private void loadDisabledTables()
1631   throws ZooKeeperConnectionException, IOException {
1632     HConnectionManager.execute(new HConnectable<Void>(getConf()) {
1633       @Override
1634       public Void connect(HConnection connection) throws IOException {
1635         ZooKeeperWatcher zkw = createZooKeeperWatcher();
1636         try {
1637           for (TableName tableName :
1638               ZKTableStateClientSideReader.getDisabledOrDisablingTables(zkw)) {
1639             disabledTables.add(tableName);
1640           }
1641         } catch (KeeperException ke) {
1642           throw new IOException(ke);
1643         } catch (InterruptedException e) {
1644           throw new InterruptedIOException();
1645         } finally {
1646           zkw.close();
1647         }
1648         return null;
1649       }
1650     });
1651   }
1652 
1653   /**
1654    * Check if the specified region's table is disabled.
1655    */
1656   private boolean isTableDisabled(HRegionInfo regionInfo) {
1657     return disabledTables.contains(regionInfo.getTable());
1658   }
1659 
1660   /**
1661    * Scan HDFS for all regions, recording their information into
1662    * regionInfoMap
1663    */
1664   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1665     Path rootDir = FSUtils.getRootDir(getConf());
1666     FileSystem fs = rootDir.getFileSystem(getConf());
1667 
1668     // list all tables from HDFS
1669     List<FileStatus> tableDirs = Lists.newArrayList();
1670 
1671     boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1672 
1673     List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1674     for (Path path : paths) {
1675       TableName tableName = FSUtils.getTableName(path);
1676        if ((!checkMetaOnly &&
1677            isTableIncluded(tableName)) ||
1678            tableName.equals(TableName.META_TABLE_NAME)) {
1679          tableDirs.add(fs.getFileStatus(path));
1680        }
1681     }
1682 
1683     // verify that version file exists
1684     if (!foundVersionFile) {
1685       errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1686           "Version file does not exist in root dir " + rootDir);
1687       if (shouldFixVersionFile()) {
1688         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1689             + " file.");
1690         setShouldRerun();
1691         FSUtils.setVersion(fs, rootDir, getConf().getInt(
1692             HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1693             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1694             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1695       }
1696     }
1697 
1698     // level 1:  <HBASE_DIR>/*
1699     List<WorkItemHdfsDir> dirs = new ArrayList<WorkItemHdfsDir>(tableDirs.size());
1700     List<Future<Void>> dirsFutures;
1701 
1702     for (FileStatus tableDir : tableDirs) {
1703       LOG.debug("Loading region dirs from " +tableDir.getPath());
1704       dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir));
1705     }
1706 
1707     // Invoke and wait for Callables to complete
1708     dirsFutures = executor.invokeAll(dirs);
1709 
1710     for(Future<Void> f: dirsFutures) {
1711       try {
1712         f.get();
1713       } catch(ExecutionException e) {
1714         LOG.warn("Could not load region dir " , e.getCause());
1715       }
1716     }
1717     errors.print("");
1718   }
1719 
1720   /**
1721    * Record the location of the hbase:meta region as found in ZooKeeper.
1722    */
1723   private boolean recordMetaRegion() throws IOException {
1724     RegionLocations rl = ((ClusterConnection)connection).locateRegion(TableName.META_TABLE_NAME,
1725         HConstants.EMPTY_START_ROW, false, false);
1726     if (rl == null) {
1727       errors.reportError(ERROR_CODE.NULL_META_REGION,
1728           "META region or some of its attributes are null.");
1729       return false;
1730     }
1731     for (HRegionLocation metaLocation : rl.getRegionLocations()) {
1732       // Check if Meta region is valid and existing
1733       if (metaLocation == null || metaLocation.getRegionInfo() == null ||
1734           metaLocation.getHostname() == null) {
1735         errors.reportError(ERROR_CODE.NULL_META_REGION,
1736             "META region or some of its attributes are null.");
1737         return false;
1738       }
1739       ServerName sn = metaLocation.getServerName();
1740       MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, EnvironmentEdgeManager.currentTime());
1741       HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1742       if (hbckInfo == null) {
1743         regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1744       } else {
1745         hbckInfo.metaEntry = m;
1746       }
1747     }
1748     return true;
1749   }
1750 
1751   private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1752     return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1753       @Override
1754       public void abort(String why, Throwable e) {
1755         LOG.error(why, e);
1756         System.exit(1);
1757       }
1758 
1759       @Override
1760       public boolean isAborted() {
1761         return false;
1762       }
1763 
1764     });
1765   }
1766 
1767   private ServerName getMetaRegionServerName(int replicaId)
1768   throws IOException, KeeperException {
1769     ZooKeeperWatcher zkw = createZooKeeperWatcher();
1770     ServerName sn = null;
1771     try {
1772       sn = new MetaTableLocator().getMetaRegionLocation(zkw, replicaId);
1773     } finally {
1774       zkw.close();
1775     }
1776     return sn;
1777   }
1778 
1779   /**
1780    * Contacts each regionserver and fetches metadata about regions.
1781    * @param regionServerList - the list of region servers to connect to
1782    * @throws IOException if a remote or network exception occurs
1783    */
1784   void processRegionServers(Collection<ServerName> regionServerList)
1785     throws IOException, InterruptedException {
1786 
1787     List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1788     List<Future<Void>> workFutures;
1789 
1790     // loop to contact each region server in parallel
1791     for (ServerName rsinfo: regionServerList) {
1792       workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1793     }
1794 
1795     workFutures = executor.invokeAll(workItems);
1796 
1797     for(int i=0; i<workFutures.size(); i++) {
1798       WorkItemRegion item = workItems.get(i);
1799       Future<Void> f = workFutures.get(i);
1800       try {
1801         f.get();
1802       } catch(ExecutionException e) {
1803         LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1804             e.getCause());
1805       }
1806     }
1807   }
1808 
1809   /**
1810    * Check consistency of all regions that have been found in previous phases.
1811    */
1812   private void checkAndFixConsistency()
1813   throws IOException, KeeperException, InterruptedException {
1814 	  // Divide the checks in two phases. One for default/primary replicas and another
1815 	  // for the non-primary ones. Keeps code cleaner this way.
1816     List<CheckRegionConsistencyWorkItem> workItems =
1817         new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1818     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1819       if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1820         workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1821       }
1822     }
1823     checkRegionConsistencyConcurrently(workItems);
1824 
1825     boolean prevHdfsCheck = shouldCheckHdfs();
1826     setCheckHdfs(false); //replicas don't have any hdfs data
1827     // Run a pass over the replicas and fix any assignment issues that exist on the currently
1828     // deployed/undeployed replicas.
1829     List<CheckRegionConsistencyWorkItem> replicaWorkItems =
1830         new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1831     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1832       if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1833         replicaWorkItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1834       }
1835     }
1836     checkRegionConsistencyConcurrently(replicaWorkItems);
1837     setCheckHdfs(prevHdfsCheck);
1838 
1839     // If some regions is skipped during checkRegionConsistencyConcurrently() phase, we might
1840     // not get accurate state of the hbase if continuing. The config here allows users to tune
1841     // the tolerance of number of skipped region.
1842     // TODO: evaluate the consequence to continue the hbck operation without config.
1843     int terminateThreshold =  getConf().getInt("hbase.hbck.skipped.regions.limit", 0);
1844     int numOfSkippedRegions = skippedRegions.size();
1845     if (numOfSkippedRegions > 0 && numOfSkippedRegions > terminateThreshold) {
1846       throw new IOException(numOfSkippedRegions
1847         + " region(s) could not be checked or repaired.  See logs for detail.");
1848     }
1849   }
1850 
1851   /**
1852    * Check consistency of all regions using mulitple threads concurrently.
1853    */
1854   private void checkRegionConsistencyConcurrently(
1855     final List<CheckRegionConsistencyWorkItem> workItems)
1856     throws IOException, KeeperException, InterruptedException {
1857     if (workItems.isEmpty()) {
1858       return;  // nothing to check
1859     }
1860 
1861     List<Future<Void>> workFutures = executor.invokeAll(workItems);
1862     for(Future<Void> f: workFutures) {
1863       try {
1864         f.get();
1865       } catch(ExecutionException e1) {
1866         LOG.warn("Could not check region consistency " , e1.getCause());
1867         if (e1.getCause() instanceof IOException) {
1868           throw (IOException)e1.getCause();
1869         } else if (e1.getCause() instanceof KeeperException) {
1870           throw (KeeperException)e1.getCause();
1871         } else if (e1.getCause() instanceof InterruptedException) {
1872           throw (InterruptedException)e1.getCause();
1873         } else {
1874           throw new IOException(e1.getCause());
1875         }
1876       }
1877     }
1878   }
1879 
1880   class CheckRegionConsistencyWorkItem implements Callable<Void> {
1881     private final String key;
1882     private final HbckInfo hbi;
1883 
1884     CheckRegionConsistencyWorkItem(String key, HbckInfo hbi) {
1885       this.key = key;
1886       this.hbi = hbi;
1887     }
1888 
1889     @Override
1890     public synchronized Void call() throws Exception {
1891       try {
1892         checkRegionConsistency(key, hbi);
1893       } catch (Exception e) {
1894         // If the region is non-META region, skip this region and send warning/error message; if
1895         // the region is META region, we should not continue.
1896         LOG.warn("Unable to complete check or repair the region '" + hbi.getRegionNameAsString()
1897           + "'.", e);
1898         if (hbi.getHdfsHRI().isMetaRegion()) {
1899           throw e;
1900         }
1901         LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'");
1902         addSkippedRegion(hbi);
1903       }
1904       return null;
1905     }
1906   }
1907 
1908   private void addSkippedRegion(final HbckInfo hbi) {
1909     Set<String> skippedRegionNames = skippedRegions.get(hbi.getTableName());
1910     if (skippedRegionNames == null) {
1911       skippedRegionNames = new HashSet<String>();
1912     }
1913     skippedRegionNames.add(hbi.getRegionNameAsString());
1914     skippedRegions.put(hbi.getTableName(), skippedRegionNames);
1915   }
1916 
1917   private void preCheckPermission() throws IOException, AccessDeniedException {
1918     if (shouldIgnorePreCheckPermission()) {
1919       return;
1920     }
1921 
1922     Path hbaseDir = FSUtils.getRootDir(getConf());
1923     FileSystem fs = hbaseDir.getFileSystem(getConf());
1924     UserProvider userProvider = UserProvider.instantiate(getConf());
1925     UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1926     FileStatus[] files = fs.listStatus(hbaseDir);
1927     for (FileStatus file : files) {
1928       try {
1929         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1930       } catch (AccessDeniedException ace) {
1931         LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1932         errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1933           + " does not have write perms to " + file.getPath()
1934           + ". Please rerun hbck as hdfs user " + file.getOwner());
1935         throw ace;
1936       }
1937     }
1938   }
1939 
1940   /**
1941    * Deletes region from meta table
1942    */
1943   private void deleteMetaRegion(HbckInfo hi) throws IOException {
1944     deleteMetaRegion(hi.metaEntry.getRegionName());
1945   }
1946 
1947   /**
1948    * Deletes region from meta table
1949    */
1950   private void deleteMetaRegion(byte[] metaKey) throws IOException {
1951     Delete d = new Delete(metaKey);
1952     meta.delete(d);
1953     LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1954   }
1955 
1956   /**
1957    * Reset the split parent region info in meta table
1958    */
1959   private void resetSplitParent(HbckInfo hi) throws IOException {
1960     RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
1961     Delete d = new Delete(hi.metaEntry.getRegionName());
1962     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1963     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1964     mutations.add(d);
1965 
1966     HRegionInfo hri = new HRegionInfo(hi.metaEntry);
1967     hri.setOffline(false);
1968     hri.setSplit(false);
1969     Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1970     mutations.add(p);
1971 
1972     meta.mutateRow(mutations);
1973     LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
1974   }
1975 
1976   /**
1977    * This backwards-compatibility wrapper for permanently offlining a region
1978    * that should not be alive.  If the region server does not support the
1979    * "offline" method, it will use the closest unassign method instead.  This
1980    * will basically work until one attempts to disable or delete the affected
1981    * table.  The problem has to do with in-memory only master state, so
1982    * restarting the HMaster or failing over to another should fix this.
1983    */
1984   private void offline(byte[] regionName) throws IOException {
1985     String regionString = Bytes.toStringBinary(regionName);
1986     if (!rsSupportsOffline) {
1987       LOG.warn("Using unassign region " + regionString
1988           + " instead of using offline method, you should"
1989           + " restart HMaster after these repairs");
1990       admin.unassign(regionName, true);
1991       return;
1992     }
1993 
1994     // first time we assume the rs's supports #offline.
1995     try {
1996       LOG.info("Offlining region " + regionString);
1997       admin.offline(regionName);
1998     } catch (IOException ioe) {
1999       String notFoundMsg = "java.lang.NoSuchMethodException: " +
2000         "org.apache.hadoop.hbase.master.HMaster.offline([B)";
2001       if (ioe.getMessage().contains(notFoundMsg)) {
2002         LOG.warn("Using unassign region " + regionString
2003             + " instead of using offline method, you should"
2004             + " restart HMaster after these repairs");
2005         rsSupportsOffline = false; // in the future just use unassign
2006         admin.unassign(regionName, true);
2007         return;
2008       }
2009       throw ioe;
2010     }
2011   }
2012 
2013   private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
2014     undeployRegionsForHbi(hi);
2015     // undeploy replicas of the region (but only if the method is invoked for the primary)
2016     if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2017       return;
2018     }
2019     int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2020     for (int i = 1; i < numReplicas; i++) {
2021       if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
2022       HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
2023           hi.getPrimaryHRIForDeployedReplica(), i);
2024       HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2025       if (h != null) {
2026         undeployRegionsForHbi(h);
2027         //set skip checks; we undeployed it, and we don't want to evaluate this anymore
2028         //in consistency checks
2029         h.setSkipChecks(true);
2030       }
2031     }
2032   }
2033 
2034   private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
2035     for (OnlineEntry rse : hi.deployedEntries) {
2036       LOG.debug("Undeploy region "  + rse.hri + " from " + rse.hsa);
2037       try {
2038         HBaseFsckRepair.closeRegionSilentlyAndWait(connection, rse.hsa, rse.hri);
2039         offline(rse.hri.getRegionName());
2040       } catch (IOException ioe) {
2041         LOG.warn("Got exception when attempting to offline region "
2042             + Bytes.toString(rse.hri.getRegionName()), ioe);
2043       }
2044     }
2045   }
2046 
2047   /**
2048    * Attempts to undeploy a region from a region server based in information in
2049    * META.  Any operations that modify the file system should make sure that
2050    * its corresponding region is not deployed to prevent data races.
2051    *
2052    * A separate call is required to update the master in-memory region state
2053    * kept in the AssignementManager.  Because disable uses this state instead of
2054    * that found in META, we can't seem to cleanly disable/delete tables that
2055    * have been hbck fixed.  When used on a version of HBase that does not have
2056    * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master
2057    * restart or failover may be required.
2058    */
2059   private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
2060     if (hi.metaEntry == null && hi.hdfsEntry == null) {
2061       undeployRegions(hi);
2062       return;
2063     }
2064 
2065     // get assignment info and hregioninfo from meta.
2066     Get get = new Get(hi.getRegionName());
2067     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2068     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
2069     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
2070     // also get the locations of the replicas to close if the primary region is being closed
2071     if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2072       int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2073       for (int i = 0; i < numReplicas; i++) {
2074         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
2075         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
2076       }
2077     }
2078     Result r = meta.get(get);
2079     RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
2080     if (rl == null) {
2081       LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
2082           " since meta does not have handle to reach it");
2083       return;
2084     }
2085     for (HRegionLocation h : rl.getRegionLocations()) {
2086       ServerName serverName = h.getServerName();
2087       if (serverName == null) {
2088         errors.reportError("Unable to close region "
2089             + hi.getRegionNameAsString() +  " because meta does not "
2090             + "have handle to reach it.");
2091         continue;
2092       }
2093       HRegionInfo hri = h.getRegionInfo();
2094       if (hri == null) {
2095         LOG.warn("Unable to close region " + hi.getRegionNameAsString()
2096             + " because hbase:meta had invalid or missing "
2097             + HConstants.CATALOG_FAMILY_STR + ":"
2098             + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
2099             + " qualifier value.");
2100         continue;
2101       }
2102       // close the region -- close files and remove assignment
2103       HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
2104     }
2105   }
2106 
2107   private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
2108     KeeperException, InterruptedException {
2109     // If we are trying to fix the errors
2110     if (shouldFixAssignments()) {
2111       errors.print(msg);
2112       undeployRegions(hbi);
2113       setShouldRerun();
2114       HRegionInfo hri = hbi.getHdfsHRI();
2115       if (hri == null) {
2116         hri = hbi.metaEntry;
2117       }
2118       HBaseFsckRepair.fixUnassigned(admin, hri);
2119       HBaseFsckRepair.waitUntilAssigned(admin, hri);
2120 
2121       // also assign replicas if needed (do it only when this call operates on a primary replica)
2122       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
2123       int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
2124       for (int i = 1; i < replicationCount; i++) {
2125         hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
2126         HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2127         if (h != null) {
2128           undeployRegions(h);
2129           //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore
2130           //in consistency checks
2131           h.setSkipChecks(true);
2132         }
2133         HBaseFsckRepair.fixUnassigned(admin, hri);
2134         HBaseFsckRepair.waitUntilAssigned(admin, hri);
2135       }
2136 
2137     }
2138   }
2139 
2140   /**
2141    * Check a single region for consistency and correct deployment.
2142    */
2143   private void checkRegionConsistency(final String key, final HbckInfo hbi)
2144   throws IOException, KeeperException, InterruptedException {
2145 
2146 	if (hbi.isSkipChecks()) return;
2147 	String descriptiveName = hbi.toString();
2148     boolean inMeta = hbi.metaEntry != null;
2149     // In case not checking HDFS, assume the region is on HDFS
2150     boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
2151     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
2152     boolean isDeployed = !hbi.deployedOn.isEmpty();
2153     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
2154     boolean deploymentMatchesMeta =
2155       hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
2156       hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
2157     boolean splitParent =
2158       (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
2159     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
2160     boolean recentlyModified = inHdfs &&
2161       hbi.getModTime() + timelag > EnvironmentEdgeManager.currentTime();
2162 
2163     // ========== First the healthy cases =============
2164     if (hbi.containsOnlyHdfsEdits()) {
2165       return;
2166     }
2167     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
2168       return;
2169     } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
2170       LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
2171         "tabled that is not deployed");
2172       return;
2173     } else if (recentlyModified) {
2174       LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
2175       return;
2176     }
2177     // ========== Cases where the region is not in hbase:meta =============
2178     else if (!inMeta && !inHdfs && !isDeployed) {
2179       // We shouldn't have record of this region at all then!
2180       assert false : "Entry for region with no data";
2181     } else if (!inMeta && !inHdfs && isDeployed) {
2182       errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
2183           + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
2184           "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2185       if (shouldFixAssignments()) {
2186         undeployRegions(hbi);
2187       }
2188 
2189     } else if (!inMeta && inHdfs && !isDeployed) {
2190       if (hbi.isMerged()) {
2191         // This region has already been merged, the remaining hdfs file will be
2192         // cleaned by CatalogJanitor later
2193         hbi.setSkipChecks(true);
2194         LOG.info("Region " + descriptiveName
2195             + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
2196         return;
2197       }
2198       errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
2199           + descriptiveName + " on HDFS, but not listed in hbase:meta " +
2200           "or deployed on any region server");
2201       // restore region consistency of an adopted orphan
2202       if (shouldFixMeta()) {
2203         if (!hbi.isHdfsRegioninfoPresent()) {
2204           LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
2205               +  " in table integrity repair phase if -fixHdfsOrphans was" +
2206               " used.");
2207           return;
2208         }
2209 
2210         HRegionInfo hri = hbi.getHdfsHRI();
2211         TableInfo tableInfo = tablesInfo.get(hri.getTable());
2212 
2213         for (HRegionInfo region : tableInfo.getRegionsFromMeta()) {
2214           if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0
2215               && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(),
2216                 hri.getEndKey()) >= 0)
2217               && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) {
2218             if(region.isSplit() || region.isOffline()) continue;
2219             Path regionDir = hbi.getHdfsRegionDir();
2220             FileSystem fs = regionDir.getFileSystem(getConf());
2221             List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir);
2222             for (Path familyDir : familyDirs) {
2223               List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);
2224               for (Path referenceFilePath : referenceFilePaths) {
2225                 Path parentRegionDir =
2226                     StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();
2227                 if (parentRegionDir.toString().endsWith(region.getEncodedName())) {
2228                   LOG.warn(hri + " start and stop keys are in the range of " + region
2229                       + ". The region might not be cleaned up from hdfs when region " + region
2230                       + " split failed. Hence deleting from hdfs.");
2231                   HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
2232                     regionDir.getParent(), hri);
2233                   return;
2234                 }
2235               }
2236             }
2237           }
2238         }
2239 
2240         LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
2241         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2242         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2243             admin.getClusterStatus().getServers(), numReplicas);
2244 
2245         tryAssignmentRepair(hbi, "Trying to reassign region...");
2246       }
2247 
2248     } else if (!inMeta && inHdfs && isDeployed) {
2249       errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
2250           + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2251       debugLsr(hbi.getHdfsRegionDir());
2252       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2253         // for replicas, this means that we should undeploy the region (we would have
2254         // gone over the primaries and fixed meta holes in first phase under
2255         // checkAndFixConsistency; we shouldn't get the condition !inMeta at
2256         // this stage unless unwanted replica)
2257         if (shouldFixAssignments()) {
2258           undeployRegionsForHbi(hbi);
2259         }
2260       }
2261       if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2262         if (!hbi.isHdfsRegioninfoPresent()) {
2263           LOG.error("This should have been repaired in table integrity repair phase");
2264           return;
2265         }
2266 
2267         LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
2268         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2269         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2270             admin.getClusterStatus().getServers(), numReplicas);
2271         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2272       }
2273 
2274     // ========== Cases where the region is in hbase:meta =============
2275     } else if (inMeta && inHdfs && !isDeployed && splitParent) {
2276       // check whether this is an actual error, or just transient state where parent
2277       // is not cleaned
2278       if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
2279         // check that split daughters are there
2280         HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
2281         HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
2282         if (infoA != null && infoB != null) {
2283           // we already processed or will process daughters. Move on, nothing to see here.
2284           hbi.setSkipChecks(true);
2285           return;
2286         }
2287       }
2288       errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2289           + descriptiveName + " is a split parent in META, in HDFS, "
2290           + "and not deployed on any region server. This could be transient.");
2291       if (shouldFixSplitParents()) {
2292         setShouldRerun();
2293         resetSplitParent(hbi);
2294       }
2295     } else if (inMeta && !inHdfs && !isDeployed) {
2296       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2297           + descriptiveName + " found in META, but not in HDFS "
2298           + "or deployed on any region server.");
2299       if (shouldFixMeta()) {
2300         deleteMetaRegion(hbi);
2301       }
2302     } else if (inMeta && !inHdfs && isDeployed) {
2303       errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2304           + " found in META, but not in HDFS, " +
2305           "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2306       // We treat HDFS as ground truth.  Any information in meta is transient
2307       // and equivalent data can be regenerated.  So, lets unassign and remove
2308       // these problems from META.
2309       if (shouldFixAssignments()) {
2310         errors.print("Trying to fix unassigned region...");
2311         undeployRegions(hbi);
2312       }
2313       if (shouldFixMeta()) {
2314         // wait for it to complete
2315         deleteMetaRegion(hbi);
2316       }
2317     } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2318       errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2319           + " not deployed on any region server.");
2320       tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2321     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2322       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2323           "Region " + descriptiveName + " should not be deployed according " +
2324           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2325       if (shouldFixAssignments()) {
2326         errors.print("Trying to close the region " + descriptiveName);
2327         setShouldRerun();
2328         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2329       }
2330     } else if (inMeta && inHdfs && isMultiplyDeployed) {
2331       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2332           + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
2333           + " but is multiply assigned to region servers " +
2334           Joiner.on(", ").join(hbi.deployedOn));
2335       // If we are trying to fix the errors
2336       if (shouldFixAssignments()) {
2337         errors.print("Trying to fix assignment error...");
2338         setShouldRerun();
2339         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2340       }
2341     } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2342       errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2343           + descriptiveName + " listed in hbase:meta on region server " +
2344           hbi.metaEntry.regionServer + " but found on region server " +
2345           hbi.deployedOn.get(0));
2346       // If we are trying to fix the errors
2347       if (shouldFixAssignments()) {
2348         errors.print("Trying to fix assignment error...");
2349         setShouldRerun();
2350         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2351         HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2352       }
2353     } else {
2354       errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2355           " is in an unforeseen state:" +
2356           " inMeta=" + inMeta +
2357           " inHdfs=" + inHdfs +
2358           " isDeployed=" + isDeployed +
2359           " isMultiplyDeployed=" + isMultiplyDeployed +
2360           " deploymentMatchesMeta=" + deploymentMatchesMeta +
2361           " shouldBeDeployed=" + shouldBeDeployed);
2362     }
2363   }
2364 
2365   /**
2366    * Checks tables integrity. Goes over all regions and scans the tables.
2367    * Collects all the pieces for each table and checks if there are missing,
2368    * repeated or overlapping ones.
2369    * @throws IOException
2370    */
2371   SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2372     tablesInfo = new TreeMap<TableName,TableInfo> ();
2373     LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2374     for (HbckInfo hbi : regionInfoMap.values()) {
2375       // Check only valid, working regions
2376       if (hbi.metaEntry == null) {
2377         // this assumes that consistency check has run loadMetaEntry
2378         Path p = hbi.getHdfsRegionDir();
2379         if (p == null) {
2380           errors.report("No regioninfo in Meta or HDFS. " + hbi);
2381         }
2382 
2383         // TODO test.
2384         continue;
2385       }
2386       if (hbi.metaEntry.regionServer == null) {
2387         errors.detail("Skipping region because no region server: " + hbi);
2388         continue;
2389       }
2390       if (hbi.metaEntry.isOffline()) {
2391         errors.detail("Skipping region because it is offline: " + hbi);
2392         continue;
2393       }
2394       if (hbi.containsOnlyHdfsEdits()) {
2395         errors.detail("Skipping region because it only contains edits" + hbi);
2396         continue;
2397       }
2398 
2399       // Missing regionDir or over-deployment is checked elsewhere. Include
2400       // these cases in modTInfo, so we can evaluate those regions as part of
2401       // the region chain in META
2402       //if (hbi.foundRegionDir == null) continue;
2403       //if (hbi.deployedOn.size() != 1) continue;
2404       if (hbi.deployedOn.size() == 0) continue;
2405 
2406       // We should be safe here
2407       TableName tableName = hbi.metaEntry.getTable();
2408       TableInfo modTInfo = tablesInfo.get(tableName);
2409       if (modTInfo == null) {
2410         modTInfo = new TableInfo(tableName);
2411       }
2412       for (ServerName server : hbi.deployedOn) {
2413         modTInfo.addServer(server);
2414       }
2415 
2416       if (!hbi.isSkipChecks()) {
2417         modTInfo.addRegionInfo(hbi);
2418       }
2419 
2420       tablesInfo.put(tableName, modTInfo);
2421     }
2422 
2423     loadTableInfosForTablesWithNoRegion();
2424 
2425     logParallelMerge();
2426     for (TableInfo tInfo : tablesInfo.values()) {
2427       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2428       if (!tInfo.checkRegionChain(handler)) {
2429         errors.report("Found inconsistency in table " + tInfo.getName());
2430       }
2431     }
2432     return tablesInfo;
2433   }
2434 
2435   /** Loads table info's for tables that may not have been included, since there are no
2436    * regions reported for the table, but table dir is there in hdfs
2437    */
2438   private void loadTableInfosForTablesWithNoRegion() throws IOException {
2439     Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2440     for (HTableDescriptor htd : allTables.values()) {
2441       if (checkMetaOnly && !htd.isMetaTable()) {
2442         continue;
2443       }
2444 
2445       TableName tableName = htd.getTableName();
2446       if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2447         TableInfo tableInfo = new TableInfo(tableName);
2448         tableInfo.htds.add(htd);
2449         tablesInfo.put(htd.getTableName(), tableInfo);
2450       }
2451     }
2452   }
2453 
2454   /**
2455    * Merge hdfs data by moving from contained HbckInfo into targetRegionDir.
2456    * @return number of file move fixes done to merge regions.
2457    */
2458   public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2459     int fileMoves = 0;
2460     String thread = Thread.currentThread().getName();
2461     LOG.debug("[" + thread + "] Contained region dir after close and pause");
2462     debugLsr(contained.getHdfsRegionDir());
2463 
2464     // rename the contained into the container.
2465     FileSystem fs = targetRegionDir.getFileSystem(getConf());
2466     FileStatus[] dirs = null;
2467     try {
2468       dirs = fs.listStatus(contained.getHdfsRegionDir());
2469     } catch (FileNotFoundException fnfe) {
2470       // region we are attempting to merge in is not present!  Since this is a merge, there is
2471       // no harm skipping this region if it does not exist.
2472       if (!fs.exists(contained.getHdfsRegionDir())) {
2473         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2474             + " is missing. Assuming already sidelined or moved.");
2475       } else {
2476         sidelineRegionDir(fs, contained);
2477       }
2478       return fileMoves;
2479     }
2480 
2481     if (dirs == null) {
2482       if (!fs.exists(contained.getHdfsRegionDir())) {
2483         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2484             + " already sidelined.");
2485       } else {
2486         sidelineRegionDir(fs, contained);
2487       }
2488       return fileMoves;
2489     }
2490 
2491     for (FileStatus cf : dirs) {
2492       Path src = cf.getPath();
2493       Path dst =  new Path(targetRegionDir, src.getName());
2494 
2495       if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2496         // do not copy the old .regioninfo file.
2497         continue;
2498       }
2499 
2500       if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2501         // do not copy the .oldlogs files
2502         continue;
2503       }
2504 
2505       LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2506       // FileSystem.rename is inconsistent with directories -- if the
2507       // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2508       // it moves the src into the dst dir resulting in (foo/a/b).  If
2509       // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2510       for (FileStatus hfile : fs.listStatus(src)) {
2511         boolean success = fs.rename(hfile.getPath(), dst);
2512         if (success) {
2513           fileMoves++;
2514         }
2515       }
2516       LOG.debug("[" + thread + "] Sideline directory contents:");
2517       debugLsr(targetRegionDir);
2518     }
2519 
2520     // if all success.
2521     sidelineRegionDir(fs, contained);
2522     LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2523         getSidelineDir());
2524     debugLsr(contained.getHdfsRegionDir());
2525 
2526     return fileMoves;
2527   }
2528 
2529 
2530   static class WorkItemOverlapMerge implements Callable<Void> {
2531     private TableIntegrityErrorHandler handler;
2532     Collection<HbckInfo> overlapgroup;
2533 
2534     WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2535       this.handler = handler;
2536       this.overlapgroup = overlapgroup;
2537     }
2538 
2539     @Override
2540     public Void call() throws Exception {
2541       handler.handleOverlapGroup(overlapgroup);
2542       return null;
2543     }
2544   };
2545 
2546 
2547   /**
2548    * Maintain information about a particular table.
2549    */
2550   public class TableInfo {
2551     TableName tableName;
2552     TreeSet <ServerName> deployedOn;
2553 
2554     // backwards regions
2555     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2556 
2557     // sidelined big overlapped regions
2558     final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2559 
2560     // region split calculator
2561     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2562 
2563     // Histogram of different HTableDescriptors found.  Ideally there is only one!
2564     final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2565 
2566     // key = start split, values = set of splits in problem group
2567     final Multimap<byte[], HbckInfo> overlapGroups =
2568       TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2569 
2570     // list of regions derived from meta entries.
2571     private ImmutableList<HRegionInfo> regionsFromMeta = null;
2572 
2573     TableInfo(TableName name) {
2574       this.tableName = name;
2575       deployedOn = new TreeSet <ServerName>();
2576     }
2577 
2578     /**
2579      * @return descriptor common to all regions.  null if are none or multiple!
2580      */
2581     private HTableDescriptor getHTD() {
2582       if (htds.size() == 1) {
2583         return (HTableDescriptor)htds.toArray()[0];
2584       } else {
2585         LOG.error("None/Multiple table descriptors found for table '"
2586           + tableName + "' regions: " + htds);
2587       }
2588       return null;
2589     }
2590 
2591     public void addRegionInfo(HbckInfo hir) {
2592       if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2593         // end key is absolute end key, just add it.
2594         // ignore replicas other than primary for these checks
2595         if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2596         return;
2597       }
2598 
2599       // if not the absolute end key, check for cycle
2600       if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2601         errors.reportError(
2602             ERROR_CODE.REGION_CYCLE,
2603             String.format("The endkey for this region comes before the "
2604                 + "startkey, startkey=%s, endkey=%s",
2605                 Bytes.toStringBinary(hir.getStartKey()),
2606                 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2607         backwards.add(hir);
2608         return;
2609       }
2610 
2611       // main case, add to split calculator
2612       // ignore replicas other than primary for these checks
2613       if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2614     }
2615 
2616     public void addServer(ServerName server) {
2617       this.deployedOn.add(server);
2618     }
2619 
2620     public TableName getName() {
2621       return tableName;
2622     }
2623 
2624     public int getNumRegions() {
2625       return sc.getStarts().size() + backwards.size();
2626     }
2627 
2628     public synchronized ImmutableList<HRegionInfo> getRegionsFromMeta() {
2629       // lazy loaded, synchronized to ensure a single load
2630       if (regionsFromMeta == null) {
2631         List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
2632         for (HbckInfo h : HBaseFsck.this.regionInfoMap.values()) {
2633           if (tableName.equals(h.getTableName())) {
2634             if (h.metaEntry != null) {
2635               regions.add((HRegionInfo) h.metaEntry);
2636             }
2637           }
2638         }
2639         regionsFromMeta = Ordering.natural().immutableSortedCopy(regions);
2640       }
2641       
2642       return regionsFromMeta;
2643     }
2644     
2645 
2646       private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2647       ErrorReporter errors;
2648 
2649       IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2650         this.errors = errors;
2651         setTableInfo(ti);
2652       }
2653 
2654       @Override
2655       public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2656         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2657             "First region should start with an empty key.  You need to "
2658             + " create a new region and regioninfo in HDFS to plug the hole.",
2659             getTableInfo(), hi);
2660       }
2661 
2662       @Override
2663       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2664         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2665             "Last region should end with an empty key. You need to "
2666                 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2667       }
2668 
2669       @Override
2670       public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2671         errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2672             "Region has the same start and end key.", getTableInfo(), hi);
2673       }
2674 
2675       @Override
2676       public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2677         byte[] key = r1.getStartKey();
2678         // dup start key
2679         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2680             "Multiple regions have the same startkey: "
2681             + Bytes.toStringBinary(key), getTableInfo(), r1);
2682         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2683             "Multiple regions have the same startkey: "
2684             + Bytes.toStringBinary(key), getTableInfo(), r2);
2685       }
2686 
2687       @Override
2688       public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2689         errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2690             "There is an overlap in the region chain.",
2691             getTableInfo(), hi1, hi2);
2692       }
2693 
2694       @Override
2695       public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2696         errors.reportError(
2697             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2698             "There is a hole in the region chain between "
2699                 + Bytes.toStringBinary(holeStart) + " and "
2700                 + Bytes.toStringBinary(holeStop)
2701                 + ".  You need to create a new .regioninfo and region "
2702                 + "dir in hdfs to plug the hole.");
2703       }
2704     };
2705 
2706     /**
2707      * This handler fixes integrity errors from hdfs information.  There are
2708      * basically three classes of integrity problems 1) holes, 2) overlaps, and
2709      * 3) invalid regions.
2710      *
2711      * This class overrides methods that fix holes and the overlap group case.
2712      * Individual cases of particular overlaps are handled by the general
2713      * overlap group merge repair case.
2714      *
2715      * If hbase is online, this forces regions offline before doing merge
2716      * operations.
2717      */
2718     private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2719       Configuration conf;
2720 
2721       boolean fixOverlaps = true;
2722 
2723       HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2724           boolean fixHoles, boolean fixOverlaps) {
2725         super(ti, errors);
2726         this.conf = conf;
2727         this.fixOverlaps = fixOverlaps;
2728         // TODO properly use fixHoles
2729       }
2730 
2731       /**
2732        * This is a special case hole -- when the first region of a table is
2733        * missing from META, HBase doesn't acknowledge the existance of the
2734        * table.
2735        */
2736       @Override
2737       public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2738         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2739             "First region should start with an empty key.  Creating a new " +
2740             "region and regioninfo in HDFS to plug the hole.",
2741             getTableInfo(), next);
2742         HTableDescriptor htd = getTableInfo().getHTD();
2743         // from special EMPTY_START_ROW to next region's startKey
2744         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2745             HConstants.EMPTY_START_ROW, next.getStartKey());
2746 
2747         // TODO test
2748         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2749         LOG.info("Table region start key was not empty.  Created new empty region: "
2750             + newRegion + " " +region);
2751         fixes++;
2752       }
2753 
2754       @Override
2755       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2756         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2757             "Last region should end with an empty key.  Creating a new "
2758                 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2759         HTableDescriptor htd = getTableInfo().getHTD();
2760         // from curEndKey to EMPTY_START_ROW
2761         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2762             HConstants.EMPTY_START_ROW);
2763 
2764         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2765         LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
2766             + " " + region);
2767         fixes++;
2768       }
2769 
2770       /**
2771        * There is a hole in the hdfs regions that violates the table integrity
2772        * rules.  Create a new empty region that patches the hole.
2773        */
2774       @Override
2775       public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2776         errors.reportError(
2777             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2778             "There is a hole in the region chain between "
2779                 + Bytes.toStringBinary(holeStartKey) + " and "
2780                 + Bytes.toStringBinary(holeStopKey)
2781                 + ".  Creating a new regioninfo and region "
2782                 + "dir in hdfs to plug the hole.");
2783         HTableDescriptor htd = getTableInfo().getHTD();
2784         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2785         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2786         LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2787         fixes++;
2788       }
2789 
2790       /**
2791        * This takes set of overlapping regions and merges them into a single
2792        * region.  This covers cases like degenerate regions, shared start key,
2793        * general overlaps, duplicate ranges, and partial overlapping regions.
2794        *
2795        * Cases:
2796        * - Clean regions that overlap
2797        * - Only .oldlogs regions (can't find start/stop range, or figure out)
2798        *
2799        * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
2800        */
2801       @Override
2802       public void handleOverlapGroup(Collection<HbckInfo> overlap)
2803           throws IOException {
2804         Preconditions.checkNotNull(overlap);
2805         Preconditions.checkArgument(overlap.size() >0);
2806 
2807         if (!this.fixOverlaps) {
2808           LOG.warn("Not attempting to repair overlaps.");
2809           return;
2810         }
2811 
2812         if (overlap.size() > maxMerge) {
2813           LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2814             "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2815           if (sidelineBigOverlaps) {
2816             // we only sideline big overlapped groups that exceeds the max number of regions to merge
2817             sidelineBigOverlaps(overlap);
2818           }
2819           return;
2820         }
2821 
2822         mergeOverlaps(overlap);
2823       }
2824 
2825       void mergeOverlaps(Collection<HbckInfo> overlap)
2826           throws IOException {
2827         String thread = Thread.currentThread().getName();
2828         LOG.info("== [" + thread + "] Merging regions into one region: "
2829           + Joiner.on(",").join(overlap));
2830         // get the min / max range and close all concerned regions
2831         Pair<byte[], byte[]> range = null;
2832         for (HbckInfo hi : overlap) {
2833           if (range == null) {
2834             range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2835           } else {
2836             if (RegionSplitCalculator.BYTES_COMPARATOR
2837                 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2838               range.setFirst(hi.getStartKey());
2839             }
2840             if (RegionSplitCalculator.BYTES_COMPARATOR
2841                 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2842               range.setSecond(hi.getEndKey());
2843             }
2844           }
2845           // need to close files so delete can happen.
2846           LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
2847           LOG.debug("[" + thread + "] Contained region dir before close");
2848           debugLsr(hi.getHdfsRegionDir());
2849           try {
2850             LOG.info("[" + thread + "] Closing region: " + hi);
2851             closeRegion(hi);
2852           } catch (IOException ioe) {
2853             LOG.warn("[" + thread + "] Was unable to close region " + hi
2854               + ".  Just continuing... ", ioe);
2855           } catch (InterruptedException e) {
2856             LOG.warn("[" + thread + "] Was unable to close region " + hi
2857               + ".  Just continuing... ", e);
2858           }
2859 
2860           try {
2861             LOG.info("[" + thread + "] Offlining region: " + hi);
2862             offline(hi.getRegionName());
2863           } catch (IOException ioe) {
2864             LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2865               + ".  Just continuing... ", ioe);
2866           }
2867         }
2868 
2869         // create new empty container region.
2870         HTableDescriptor htd = getTableInfo().getHTD();
2871         // from start key to end Key
2872         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
2873             range.getSecond());
2874         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2875         LOG.info("[" + thread + "] Created new empty container region: " +
2876             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
2877         debugLsr(region.getRegionFileSystem().getRegionDir());
2878 
2879         // all target regions are closed, should be able to safely cleanup.
2880         boolean didFix= false;
2881         Path target = region.getRegionFileSystem().getRegionDir();
2882         for (HbckInfo contained : overlap) {
2883           LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
2884           int merges = mergeRegionDirs(target, contained);
2885           if (merges > 0) {
2886             didFix = true;
2887           }
2888         }
2889         if (didFix) {
2890           fixes++;
2891         }
2892       }
2893 
2894       /**
2895        * Sideline some regions in a big overlap group so that it
2896        * will have fewer regions, and it is easier to merge them later on.
2897        *
2898        * @param bigOverlap the overlapped group with regions more than maxMerge
2899        * @throws IOException
2900        */
2901       void sidelineBigOverlaps(
2902           Collection<HbckInfo> bigOverlap) throws IOException {
2903         int overlapsToSideline = bigOverlap.size() - maxMerge;
2904         if (overlapsToSideline > maxOverlapsToSideline) {
2905           overlapsToSideline = maxOverlapsToSideline;
2906         }
2907         List<HbckInfo> regionsToSideline =
2908           RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
2909         FileSystem fs = FileSystem.get(conf);
2910         for (HbckInfo regionToSideline: regionsToSideline) {
2911           try {
2912             LOG.info("Closing region: " + regionToSideline);
2913             closeRegion(regionToSideline);
2914           } catch (IOException ioe) {
2915             LOG.warn("Was unable to close region " + regionToSideline
2916               + ".  Just continuing... ", ioe);
2917           } catch (InterruptedException e) {
2918             LOG.warn("Was unable to close region " + regionToSideline
2919               + ".  Just continuing... ", e);
2920           }
2921 
2922           try {
2923             LOG.info("Offlining region: " + regionToSideline);
2924             offline(regionToSideline.getRegionName());
2925           } catch (IOException ioe) {
2926             LOG.warn("Unable to offline region from master: " + regionToSideline
2927               + ".  Just continuing... ", ioe);
2928           }
2929 
2930           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
2931           Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
2932           if (sidelineRegionDir != null) {
2933             sidelinedRegions.put(sidelineRegionDir, regionToSideline);
2934             LOG.info("After sidelined big overlapped region: "
2935               + regionToSideline.getRegionNameAsString()
2936               + " to " + sidelineRegionDir.toString());
2937             fixes++;
2938           }
2939         }
2940       }
2941     }
2942 
2943     /**
2944      * Check the region chain (from META) of this table.  We are looking for
2945      * holes, overlaps, and cycles.
2946      * @return false if there are errors
2947      * @throws IOException
2948      */
2949     public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
2950       // When table is disabled no need to check for the region chain. Some of the regions
2951       // accidently if deployed, this below code might report some issues like missing start
2952       // or end regions or region hole in chain and may try to fix which is unwanted.
2953       if (disabledTables.contains(this.tableName)) {
2954         return true;
2955       }
2956       int originalErrorsCount = errors.getErrorList().size();
2957       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
2958       SortedSet<byte[]> splits = sc.getSplits();
2959 
2960       byte[] prevKey = null;
2961       byte[] problemKey = null;
2962 
2963       if (splits.size() == 0) {
2964         // no region for this table
2965         handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
2966       }
2967 
2968       for (byte[] key : splits) {
2969         Collection<HbckInfo> ranges = regions.get(key);
2970         if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
2971           for (HbckInfo rng : ranges) {
2972             handler.handleRegionStartKeyNotEmpty(rng);
2973           }
2974         }
2975 
2976         // check for degenerate ranges
2977         for (HbckInfo rng : ranges) {
2978           // special endkey case converts '' to null
2979           byte[] endKey = rng.getEndKey();
2980           endKey = (endKey.length == 0) ? null : endKey;
2981           if (Bytes.equals(rng.getStartKey(),endKey)) {
2982             handler.handleDegenerateRegion(rng);
2983           }
2984         }
2985 
2986         if (ranges.size() == 1) {
2987           // this split key is ok -- no overlap, not a hole.
2988           if (problemKey != null) {
2989             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2990           }
2991           problemKey = null; // fell through, no more problem.
2992         } else if (ranges.size() > 1) {
2993           // set the new problem key group name, if already have problem key, just
2994           // keep using it.
2995           if (problemKey == null) {
2996             // only for overlap regions.
2997             LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
2998             problemKey = key;
2999           }
3000           overlapGroups.putAll(problemKey, ranges);
3001 
3002           // record errors
3003           ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
3004           //  this dumb and n^2 but this shouldn't happen often
3005           for (HbckInfo r1 : ranges) {
3006             if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3007             subRange.remove(r1);
3008             for (HbckInfo r2 : subRange) {
3009               if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3010               if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
3011                 handler.handleDuplicateStartKeys(r1,r2);
3012               } else {
3013                 // overlap
3014                 handler.handleOverlapInRegionChain(r1, r2);
3015               }
3016             }
3017           }
3018 
3019         } else if (ranges.size() == 0) {
3020           if (problemKey != null) {
3021             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3022           }
3023           problemKey = null;
3024 
3025           byte[] holeStopKey = sc.getSplits().higher(key);
3026           // if higher key is null we reached the top.
3027           if (holeStopKey != null) {
3028             // hole
3029             handler.handleHoleInRegionChain(key, holeStopKey);
3030           }
3031         }
3032         prevKey = key;
3033       }
3034 
3035       // When the last region of a table is proper and having an empty end key, 'prevKey'
3036       // will be null.
3037       if (prevKey != null) {
3038         handler.handleRegionEndKeyNotEmpty(prevKey);
3039       }
3040 
3041       // TODO fold this into the TableIntegrityHandler
3042       if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
3043         boolean ok = handleOverlapsParallel(handler, prevKey);
3044         if (!ok) {
3045           return false;
3046         }
3047       } else {
3048         for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3049           handler.handleOverlapGroup(overlap);
3050         }
3051       }
3052 
3053       if (details) {
3054         // do full region split map dump
3055         errors.print("---- Table '"  +  this.tableName
3056             + "': region split map");
3057         dump(splits, regions);
3058         errors.print("---- Table '"  +  this.tableName
3059             + "': overlap groups");
3060         dumpOverlapProblems(overlapGroups);
3061         errors.print("There are " + overlapGroups.keySet().size()
3062             + " overlap groups with " + overlapGroups.size()
3063             + " overlapping regions");
3064       }
3065       if (!sidelinedRegions.isEmpty()) {
3066         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
3067         errors.print("---- Table '"  +  this.tableName
3068             + "': sidelined big overlapped regions");
3069         dumpSidelinedRegions(sidelinedRegions);
3070       }
3071       return errors.getErrorList().size() == originalErrorsCount;
3072     }
3073 
3074     private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
3075         throws IOException {
3076       // we parallelize overlap handler for the case we have lots of groups to fix.  We can
3077       // safely assume each group is independent.
3078       List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
3079       List<Future<Void>> rets;
3080       for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3081         //
3082         merges.add(new WorkItemOverlapMerge(overlap, handler));
3083       }
3084       try {
3085         rets = executor.invokeAll(merges);
3086       } catch (InterruptedException e) {
3087         LOG.error("Overlap merges were interrupted", e);
3088         return false;
3089       }
3090       for(int i=0; i<merges.size(); i++) {
3091         WorkItemOverlapMerge work = merges.get(i);
3092         Future<Void> f = rets.get(i);
3093         try {
3094           f.get();
3095         } catch(ExecutionException e) {
3096           LOG.warn("Failed to merge overlap group" + work, e.getCause());
3097         } catch (InterruptedException e) {
3098           LOG.error("Waiting for overlap merges was interrupted", e);
3099           return false;
3100         }
3101       }
3102       return true;
3103     }
3104 
3105     /**
3106      * This dumps data in a visually reasonable way for visual debugging
3107      *
3108      * @param splits
3109      * @param regions
3110      */
3111     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
3112       // we display this way because the last end key should be displayed as well.
3113       StringBuilder sb = new StringBuilder();
3114       for (byte[] k : splits) {
3115         sb.setLength(0); // clear out existing buffer, if any.
3116         sb.append(Bytes.toStringBinary(k) + ":\t");
3117         for (HbckInfo r : regions.get(k)) {
3118           sb.append("[ "+ r.toString() + ", "
3119               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
3120         }
3121         errors.print(sb.toString());
3122       }
3123     }
3124   }
3125 
3126   public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
3127     // we display this way because the last end key should be displayed as
3128     // well.
3129     for (byte[] k : regions.keySet()) {
3130       errors.print(Bytes.toStringBinary(k) + ":");
3131       for (HbckInfo r : regions.get(k)) {
3132         errors.print("[ " + r.toString() + ", "
3133             + Bytes.toStringBinary(r.getEndKey()) + "]");
3134       }
3135       errors.print("----");
3136     }
3137   }
3138 
3139   public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
3140     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
3141       TableName tableName = entry.getValue().getTableName();
3142       Path path = entry.getKey();
3143       errors.print("This sidelined region dir should be bulk loaded: "
3144         + path.toString());
3145       errors.print("Bulk load command looks like: "
3146         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
3147         + path.toUri().getPath() + " "+ tableName);
3148     }
3149   }
3150 
3151   public Multimap<byte[], HbckInfo> getOverlapGroups(
3152       TableName table) {
3153     TableInfo ti = tablesInfo.get(table);
3154     return ti.overlapGroups;
3155   }
3156 
3157   /**
3158    * Return a list of user-space table names whose metadata have not been
3159    * modified in the last few milliseconds specified by timelag
3160    * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
3161    * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
3162    * milliseconds specified by timelag, then the table is a candidate to be returned.
3163    * @return tables that have not been modified recently
3164    * @throws IOException if an error is encountered
3165    */
3166   HTableDescriptor[] getTables(AtomicInteger numSkipped) {
3167     List<TableName> tableNames = new ArrayList<TableName>();
3168     long now = EnvironmentEdgeManager.currentTime();
3169 
3170     for (HbckInfo hbi : regionInfoMap.values()) {
3171       MetaEntry info = hbi.metaEntry;
3172 
3173       // if the start key is zero, then we have found the first region of a table.
3174       // pick only those tables that were not modified in the last few milliseconds.
3175       if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
3176         if (info.modTime + timelag < now) {
3177           tableNames.add(info.getTable());
3178         } else {
3179           numSkipped.incrementAndGet(); // one more in-flux table
3180         }
3181       }
3182     }
3183     return getHTableDescriptors(tableNames);
3184   }
3185 
3186   HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
3187     HTableDescriptor[] htd = new HTableDescriptor[0];
3188     Admin admin = null;
3189     try {
3190       LOG.info("getHTableDescriptors == tableNames => " + tableNames);
3191       admin = new HBaseAdmin(getConf());
3192       htd = admin.getTableDescriptorsByTableName(tableNames);
3193     } catch (IOException e) {
3194       LOG.debug("Exception getting table descriptors", e);
3195     } finally {
3196       if (admin != null) {
3197         try {
3198           admin.close();
3199         } catch (IOException e) {
3200           LOG.debug("Exception closing HBaseAdmin", e);
3201         }
3202       }
3203     }
3204     return htd;
3205   }
3206 
3207   /**
3208    * Gets the entry in regionInfo corresponding to the the given encoded
3209    * region name. If the region has not been seen yet, a new entry is added
3210    * and returned.
3211    */
3212   private synchronized HbckInfo getOrCreateInfo(String name) {
3213     HbckInfo hbi = regionInfoMap.get(name);
3214     if (hbi == null) {
3215       hbi = new HbckInfo(null);
3216       regionInfoMap.put(name, hbi);
3217     }
3218     return hbi;
3219   }
3220 
3221   private void checkAndFixTableLocks() throws IOException {
3222     ZooKeeperWatcher zkw = createZooKeeperWatcher();
3223 
3224     try {
3225       TableLockChecker checker = new TableLockChecker(zkw, errors);
3226       checker.checkTableLocks();
3227 
3228       if (this.fixTableLocks) {
3229         checker.fixExpiredTableLocks();
3230       }
3231     } finally {
3232       zkw.close();
3233     }
3234   }
3235 
3236   /**
3237    * Check whether a orphaned table ZNode exists and fix it if requested.
3238    * @throws IOException
3239    * @throws KeeperException
3240    * @throws InterruptedException
3241    */
3242   private void checkAndFixOrphanedTableZNodes()
3243       throws IOException, KeeperException, InterruptedException {
3244     ZooKeeperWatcher zkw = createZooKeeperWatcher();
3245 
3246     try {
3247       Set<TableName> enablingTables = ZKTableStateClientSideReader.getEnablingTables(zkw);
3248       String msg;
3249       TableInfo tableInfo;
3250 
3251       for (TableName tableName : enablingTables) {
3252         // Check whether the table exists in hbase
3253         tableInfo = tablesInfo.get(tableName);
3254         if (tableInfo != null) {
3255           // Table exists.  This table state is in transit.  No problem for this table.
3256           continue;
3257         }
3258 
3259         msg = "Table " + tableName + " not found in hbase:meta. Orphaned table ZNode found.";
3260         LOG.warn(msg);
3261         orphanedTableZNodes.add(tableName);
3262         errors.reportError(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY, msg);
3263       }
3264 
3265       if (orphanedTableZNodes.size() > 0 && this.fixTableZNodes) {
3266         ZKTableStateManager zkTableStateMgr = new ZKTableStateManager(zkw);
3267 
3268         for (TableName tableName : orphanedTableZNodes) {
3269           try {
3270             // Set the table state to be disabled so that if we made mistake, we can trace
3271             // the history and figure it out.
3272             // Another choice is to call checkAndRemoveTableState() to delete the orphaned ZNode.
3273             // Both approaches works.
3274             zkTableStateMgr.setTableState(tableName, ZooKeeperProtos.Table.State.DISABLED);
3275           } catch (CoordinatedStateException e) {
3276             // This exception should not happen here
3277             LOG.error(
3278               "Got a CoordinatedStateException while fixing the ENABLING table znode " + tableName,
3279               e);
3280           }
3281         }
3282       }
3283     } finally {
3284       zkw.close();
3285     }
3286   }
3287 
3288   /**
3289     * Check values in regionInfo for hbase:meta
3290     * Check if zero or more than one regions with hbase:meta are found.
3291     * If there are inconsistencies (i.e. zero or more than one regions
3292     * pretend to be holding the hbase:meta) try to fix that and report an error.
3293     * @throws IOException from HBaseFsckRepair functions
3294     * @throws KeeperException
3295     * @throws InterruptedException
3296     */
3297   boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
3298     Map<Integer, HbckInfo> metaRegions = new HashMap<Integer, HbckInfo>();
3299     for (HbckInfo value : regionInfoMap.values()) {
3300       if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
3301         metaRegions.put(value.getReplicaId(), value);
3302       }
3303     }
3304     int metaReplication = admin.getTableDescriptor(TableName.META_TABLE_NAME)
3305         .getRegionReplication();
3306     boolean noProblem = true;
3307     // There will be always entries in regionInfoMap corresponding to hbase:meta & its replicas
3308     // Check the deployed servers. It should be exactly one server for each replica.
3309     for (int i = 0; i < metaReplication; i++) {
3310       HbckInfo metaHbckInfo = metaRegions.remove(i);
3311       List<ServerName> servers = new ArrayList<ServerName>();
3312       if (metaHbckInfo != null) {
3313         servers = metaHbckInfo.deployedOn;
3314       }
3315       if (servers.size() != 1) {
3316         noProblem = false;
3317         if (servers.size() == 0) {
3318           assignMetaReplica(i);
3319         } else if (servers.size() > 1) {
3320           errors
3321           .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId " +
3322                        metaHbckInfo.getReplicaId() + " is found on more than one region.");
3323           if (shouldFixAssignments()) {
3324             errors.print("Trying to fix a problem with hbase:meta, replicaId " +
3325                          metaHbckInfo.getReplicaId() +"..");
3326             setShouldRerun();
3327             // try fix it (treat is a dupe assignment)
3328             HBaseFsckRepair.fixMultiAssignment(connection, metaHbckInfo.metaEntry, servers);
3329           }
3330         }
3331       }
3332     }
3333     // unassign whatever is remaining in metaRegions. They are excess replicas.
3334     for (Map.Entry<Integer, HbckInfo> entry : metaRegions.entrySet()) {
3335       noProblem = false;
3336       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
3337           "hbase:meta replicas are deployed in excess. Configured " + metaReplication +
3338           ", deployed " + metaRegions.size());
3339       if (shouldFixAssignments()) {
3340         errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() +
3341             " of hbase:meta..");
3342         setShouldRerun();
3343         unassignMetaReplica(entry.getValue());
3344       }
3345     }
3346     // if noProblem is false, rerun hbck with hopefully fixed META
3347     // if noProblem is true, no errors, so continue normally
3348     return noProblem;
3349   }
3350 
3351   private void unassignMetaReplica(HbckInfo hi) throws IOException, InterruptedException,
3352   KeeperException {
3353     undeployRegions(hi);
3354     ZooKeeperWatcher zkw = createZooKeeperWatcher();
3355     try {
3356       ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(hi.metaEntry.getReplicaId()));
3357     } finally {
3358       zkw.close();
3359     }
3360   }
3361 
3362   private void assignMetaReplica(int replicaId)
3363       throws IOException, KeeperException, InterruptedException {
3364     errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " +
3365         replicaId +" is not found on any region.");
3366     if (shouldFixAssignments()) {
3367       errors.print("Trying to fix a problem with hbase:meta..");
3368       setShouldRerun();
3369       // try to fix it (treat it as unassigned region)
3370       HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
3371           HRegionInfo.FIRST_META_REGIONINFO, replicaId);
3372       HBaseFsckRepair.fixUnassigned(admin, h);
3373       HBaseFsckRepair.waitUntilAssigned(admin, h);
3374     }
3375   }
3376 
3377   /**
3378    * Scan hbase:meta, adding all regions found to the regionInfo map.
3379    * @throws IOException if an error is encountered
3380    */
3381   boolean loadMetaEntries() throws IOException {
3382     MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
3383       int countRecord = 1;
3384 
3385       // comparator to sort KeyValues with latest modtime
3386       final Comparator<Cell> comp = new Comparator<Cell>() {
3387         @Override
3388         public int compare(Cell k1, Cell k2) {
3389           return Long.compare(k1.getTimestamp(), k2.getTimestamp());
3390         }
3391       };
3392 
3393       @Override
3394       public boolean processRow(Result result) throws IOException {
3395         try {
3396 
3397           // record the latest modification of this META record
3398           long ts =  Collections.max(result.listCells(), comp).getTimestamp();
3399           RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
3400           if (rl == null) {
3401             emptyRegionInfoQualifiers.add(result);
3402             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3403               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3404             return true;
3405           }
3406           ServerName sn = null;
3407           if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
3408               rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
3409             emptyRegionInfoQualifiers.add(result);
3410             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3411               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3412             return true;
3413           }
3414           HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
3415           if (!(isTableIncluded(hri.getTable())
3416               || hri.isMetaRegion())) {
3417             return true;
3418           }
3419           PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
3420           for (HRegionLocation h : rl.getRegionLocations()) {
3421             if (h == null || h.getRegionInfo() == null) {
3422               continue;
3423             }
3424             sn = h.getServerName();
3425             hri = h.getRegionInfo();
3426 
3427             MetaEntry m = null;
3428             if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
3429               m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
3430             } else {
3431               m = new MetaEntry(hri, sn, ts, null, null);
3432             }
3433             HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
3434             if (previous == null) {
3435               regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
3436             } else if (previous.metaEntry == null) {
3437               previous.metaEntry = m;
3438             } else {
3439               throw new IOException("Two entries in hbase:meta are same " + previous);
3440             }
3441           }
3442           PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
3443           for (HRegionInfo mergeRegion : new HRegionInfo[] {
3444               mergeRegions.getFirst(), mergeRegions.getSecond() }) {
3445             if (mergeRegion != null) {
3446               // This region is already been merged
3447               HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
3448               hbInfo.setMerged(true);
3449             }
3450           }
3451 
3452           // show proof of progress to the user, once for every 100 records.
3453           if (countRecord % 100 == 0) {
3454             errors.progress();
3455           }
3456           countRecord++;
3457           return true;
3458         } catch (RuntimeException e) {
3459           LOG.error("Result=" + result);
3460           throw e;
3461         }
3462       }
3463     };
3464     if (!checkMetaOnly) {
3465       // Scan hbase:meta to pick up user regions
3466       MetaScanner.metaScan(connection, visitor);
3467     }
3468 
3469     errors.print("");
3470     return true;
3471   }
3472 
3473   /**
3474    * Stores the regioninfo entries scanned from META
3475    */
3476   static class MetaEntry extends HRegionInfo {
3477     ServerName regionServer;   // server hosting this region
3478     long modTime;          // timestamp of most recent modification metadata
3479     HRegionInfo splitA, splitB; //split daughters
3480 
3481     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3482       this(rinfo, regionServer, modTime, null, null);
3483     }
3484 
3485     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3486         HRegionInfo splitA, HRegionInfo splitB) {
3487       super(rinfo);
3488       this.regionServer = regionServer;
3489       this.modTime = modTime;
3490       this.splitA = splitA;
3491       this.splitB = splitB;
3492     }
3493 
3494     @Override
3495     public boolean equals(Object o) {
3496       boolean superEq = super.equals(o);
3497       if (!superEq) {
3498         return superEq;
3499       }
3500 
3501       MetaEntry me = (MetaEntry) o;
3502       if (!regionServer.equals(me.regionServer)) {
3503         return false;
3504       }
3505       return (modTime == me.modTime);
3506     }
3507 
3508     @Override
3509     public int hashCode() {
3510       int hash = Arrays.hashCode(getRegionName());
3511       hash ^= getRegionId();
3512       hash ^= Arrays.hashCode(getStartKey());
3513       hash ^= Arrays.hashCode(getEndKey());
3514       hash ^= Boolean.valueOf(isOffline()).hashCode();
3515       hash ^= getTable().hashCode();
3516       if (regionServer != null) {
3517         hash ^= regionServer.hashCode();
3518       }
3519       hash ^= modTime;
3520       return hash;
3521     }
3522   }
3523 
3524   /**
3525    * Stores the regioninfo entries from HDFS
3526    */
3527   static class HdfsEntry {
3528     HRegionInfo hri;
3529     Path hdfsRegionDir = null;
3530     long hdfsRegionDirModTime  = 0;
3531     boolean hdfsRegioninfoFilePresent = false;
3532     boolean hdfsOnlyEdits = false;
3533   }
3534 
3535   /**
3536    * Stores the regioninfo retrieved from Online region servers.
3537    */
3538   static class OnlineEntry {
3539     HRegionInfo hri;
3540     ServerName hsa;
3541 
3542     @Override
3543     public String toString() {
3544       return hsa.toString() + ";" + hri.getRegionNameAsString();
3545     }
3546   }
3547 
3548   /**
3549    * Maintain information about a particular region.  It gathers information
3550    * from three places -- HDFS, META, and region servers.
3551    */
3552   public static class HbckInfo implements KeyRange {
3553     private MetaEntry metaEntry = null; // info in META
3554     private HdfsEntry hdfsEntry = null; // info in HDFS
3555     private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
3556     private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
3557     private boolean skipChecks = false; // whether to skip further checks to this region info.
3558     private boolean isMerged = false;// whether this region has already been merged into another one
3559     private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3560     private HRegionInfo primaryHRIForDeployedReplica = null;
3561 
3562     HbckInfo(MetaEntry metaEntry) {
3563       this.metaEntry = metaEntry;
3564     }
3565 
3566     public synchronized int getReplicaId() {
3567       return metaEntry != null? metaEntry.getReplicaId(): deployedReplicaId;
3568     }
3569 
3570     public synchronized void addServer(HRegionInfo hri, ServerName server) {
3571       OnlineEntry rse = new OnlineEntry() ;
3572       rse.hri = hri;
3573       rse.hsa = server;
3574       this.deployedEntries.add(rse);
3575       this.deployedOn.add(server);
3576       // save the replicaId that we see deployed in the cluster
3577       this.deployedReplicaId = hri.getReplicaId();
3578       this.primaryHRIForDeployedReplica =
3579           RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3580     }
3581 
3582     @Override
3583     public synchronized String toString() {
3584       StringBuilder sb = new StringBuilder();
3585       sb.append("{ meta => ");
3586       sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3587       sb.append( ", hdfs => " + getHdfsRegionDir());
3588       sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3589       sb.append( ", replicaId => " + getReplicaId());
3590       sb.append(" }");
3591       return sb.toString();
3592     }
3593 
3594     @Override
3595     public byte[] getStartKey() {
3596       if (this.metaEntry != null) {
3597         return this.metaEntry.getStartKey();
3598       } else if (this.hdfsEntry != null) {
3599         return this.hdfsEntry.hri.getStartKey();
3600       } else {
3601         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3602         return null;
3603       }
3604     }
3605 
3606     @Override
3607     public byte[] getEndKey() {
3608       if (this.metaEntry != null) {
3609         return this.metaEntry.getEndKey();
3610       } else if (this.hdfsEntry != null) {
3611         return this.hdfsEntry.hri.getEndKey();
3612       } else {
3613         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3614         return null;
3615       }
3616     }
3617 
3618     public TableName getTableName() {
3619       if (this.metaEntry != null) {
3620         return this.metaEntry.getTable();
3621       } else if (this.hdfsEntry != null) {
3622         // we are only guaranteed to have a path and not an HRI for hdfsEntry,
3623         // so we get the name from the Path
3624         Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3625         return FSUtils.getTableName(tableDir);
3626       } else {
3627         // return the info from the first online/deployed hri
3628         for (OnlineEntry e : deployedEntries) {
3629           return e.hri.getTable();
3630         }
3631         return null;
3632       }
3633     }
3634 
3635     public String getRegionNameAsString() {
3636       if (metaEntry != null) {
3637         return metaEntry.getRegionNameAsString();
3638       } else if (hdfsEntry != null) {
3639         if (hdfsEntry.hri != null) {
3640           return hdfsEntry.hri.getRegionNameAsString();
3641         }
3642       } else {
3643         // return the info from the first online/deployed hri
3644         for (OnlineEntry e : deployedEntries) {
3645           return e.hri.getRegionNameAsString();
3646         }
3647       }
3648       return null;
3649     }
3650 
3651     public byte[] getRegionName() {
3652       if (metaEntry != null) {
3653         return metaEntry.getRegionName();
3654       } else if (hdfsEntry != null) {
3655         return hdfsEntry.hri.getRegionName();
3656       } else {
3657         // return the info from the first online/deployed hri
3658         for (OnlineEntry e : deployedEntries) {
3659           return e.hri.getRegionName();
3660         }
3661         return null;
3662       }
3663     }
3664 
3665     public HRegionInfo getPrimaryHRIForDeployedReplica() {
3666       return primaryHRIForDeployedReplica;
3667     }
3668 
3669     Path getHdfsRegionDir() {
3670       if (hdfsEntry == null) {
3671         return null;
3672       }
3673       return hdfsEntry.hdfsRegionDir;
3674     }
3675 
3676     boolean containsOnlyHdfsEdits() {
3677       if (hdfsEntry == null) {
3678         return false;
3679       }
3680       return hdfsEntry.hdfsOnlyEdits;
3681     }
3682 
3683     boolean isHdfsRegioninfoPresent() {
3684       if (hdfsEntry == null) {
3685         return false;
3686       }
3687       return hdfsEntry.hdfsRegioninfoFilePresent;
3688     }
3689 
3690     long getModTime() {
3691       if (hdfsEntry == null) {
3692         return 0;
3693       }
3694       return hdfsEntry.hdfsRegionDirModTime;
3695     }
3696 
3697     HRegionInfo getHdfsHRI() {
3698       if (hdfsEntry == null) {
3699         return null;
3700       }
3701       return hdfsEntry.hri;
3702     }
3703 
3704     public void setSkipChecks(boolean skipChecks) {
3705       this.skipChecks = skipChecks;
3706     }
3707 
3708     public boolean isSkipChecks() {
3709       return skipChecks;
3710     }
3711 
3712     public void setMerged(boolean isMerged) {
3713       this.isMerged = isMerged;
3714     }
3715 
3716     public boolean isMerged() {
3717       return this.isMerged;
3718     }
3719   }
3720 
3721   final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3722     @Override
3723     public int compare(HbckInfo l, HbckInfo r) {
3724       if (l == r) {
3725         // same instance
3726         return 0;
3727       }
3728 
3729       int tableCompare = l.getTableName().compareTo(r.getTableName());
3730       if (tableCompare != 0) {
3731         return tableCompare;
3732       }
3733 
3734       int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3735           l.getStartKey(), r.getStartKey());
3736       if (startComparison != 0) {
3737         return startComparison;
3738       }
3739 
3740       // Special case for absolute endkey
3741       byte[] endKey = r.getEndKey();
3742       endKey = (endKey.length == 0) ? null : endKey;
3743       byte[] endKey2 = l.getEndKey();
3744       endKey2 = (endKey2.length == 0) ? null : endKey2;
3745       int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3746           endKey2,  endKey);
3747 
3748       if (endComparison != 0) {
3749         return endComparison;
3750       }
3751 
3752       // use regionId as tiebreaker.
3753       // Null is considered after all possible values so make it bigger.
3754       if (l.hdfsEntry == null && r.hdfsEntry == null) {
3755         return 0;
3756       }
3757       if (l.hdfsEntry == null && r.hdfsEntry != null) {
3758         return 1;
3759       }
3760       // l.hdfsEntry must not be null
3761       if (r.hdfsEntry == null) {
3762         return -1;
3763       }
3764       // both l.hdfsEntry and r.hdfsEntry must not be null.
3765       return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3766     }
3767   };
3768 
3769   /**
3770    * Prints summary of all tables found on the system.
3771    */
3772   private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3773     StringBuilder sb = new StringBuilder();
3774     int numOfSkippedRegions;
3775     errors.print("Summary:");
3776     for (TableInfo tInfo : tablesInfo.values()) {
3777       numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName())) ?
3778           skippedRegions.get(tInfo.getName()).size() : 0;
3779 
3780       if (errors.tableHasErrors(tInfo)) {
3781         errors.print("Table " + tInfo.getName() + " is inconsistent.");
3782       } else if (numOfSkippedRegions > 0){
3783         errors.print("Table " + tInfo.getName() + " is okay (with "
3784           + numOfSkippedRegions + " skipped regions).");
3785       }
3786       else {
3787         errors.print("Table " + tInfo.getName() + " is okay.");
3788       }
3789       errors.print("    Number of regions: " + tInfo.getNumRegions());
3790       if (numOfSkippedRegions > 0) {
3791         Set<String> skippedRegionStrings = skippedRegions.get(tInfo.getName());
3792         System.out.println("    Number of skipped regions: " + numOfSkippedRegions);
3793         System.out.println("      List of skipped regions:");
3794         for(String sr : skippedRegionStrings) {
3795           System.out.println("        " + sr);
3796         }
3797       }
3798       sb.setLength(0); // clear out existing buffer, if any.
3799       sb.append("    Deployed on: ");
3800       for (ServerName server : tInfo.deployedOn) {
3801         sb.append(" " + server.toString());
3802       }
3803       errors.print(sb.toString());
3804     }
3805   }
3806 
3807   static ErrorReporter getErrorReporter(
3808       final Configuration conf) throws ClassNotFoundException {
3809     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3810     return ReflectionUtils.newInstance(reporter, conf);
3811   }
3812 
3813   public interface ErrorReporter {
3814     enum ERROR_CODE {
3815       UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3816       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
3817       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3818       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3819       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3820       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3821       WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, ORPHANED_ZK_TABLE_ENTRY, BOUNDARIES_ERROR
3822     }
3823     void clear();
3824     void report(String message);
3825     void reportError(String message);
3826     void reportError(ERROR_CODE errorCode, String message);
3827     void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3828     void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3829     void reportError(
3830       ERROR_CODE errorCode,
3831       String message,
3832       TableInfo table,
3833       HbckInfo info1,
3834       HbckInfo info2
3835     );
3836     int summarize();
3837     void detail(String details);
3838     ArrayList<ERROR_CODE> getErrorList();
3839     void progress();
3840     void print(String message);
3841     void resetErrors();
3842     boolean tableHasErrors(TableInfo table);
3843   }
3844 
3845   static class PrintingErrorReporter implements ErrorReporter {
3846     public int errorCount = 0;
3847     private int showProgress;
3848     // How frequently calls to progress() will create output
3849     private static final int progressThreshold = 100;
3850 
3851     Set<TableInfo> errorTables = new HashSet<TableInfo>();
3852 
3853     // for use by unit tests to verify which errors were discovered
3854     private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3855 
3856     @Override
3857     public void clear() {
3858       errorTables.clear();
3859       errorList.clear();
3860       errorCount = 0;
3861     }
3862 
3863     @Override
3864     public synchronized void reportError(ERROR_CODE errorCode, String message) {
3865       if (errorCode == ERROR_CODE.WRONG_USAGE) {
3866         System.err.println(message);
3867         return;
3868       }
3869 
3870       errorList.add(errorCode);
3871       if (!getSUMMARY()) {
3872         System.out.println("ERROR: " + message);
3873       }
3874       errorCount++;
3875       showProgress = 0;
3876     }
3877 
3878     @Override
3879     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
3880       errorTables.add(table);
3881       reportError(errorCode, message);
3882     }
3883 
3884     @Override
3885     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3886                                          HbckInfo info) {
3887       errorTables.add(table);
3888       String reference = "(region " + info.getRegionNameAsString() + ")";
3889       reportError(errorCode, reference + " " + message);
3890     }
3891 
3892     @Override
3893     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3894                                          HbckInfo info1, HbckInfo info2) {
3895       errorTables.add(table);
3896       String reference = "(regions " + info1.getRegionNameAsString()
3897           + " and " + info2.getRegionNameAsString() + ")";
3898       reportError(errorCode, reference + " " + message);
3899     }
3900 
3901     @Override
3902     public synchronized void reportError(String message) {
3903       reportError(ERROR_CODE.UNKNOWN, message);
3904     }
3905 
3906     /**
3907      * Report error information, but do not increment the error count.  Intended for cases
3908      * where the actual error would have been reported previously.
3909      * @param message
3910      */
3911     @Override
3912     public synchronized void report(String message) {
3913       if (!getSUMMARY()) {
3914         System.out.println("ERROR: " + message);
3915       }
3916       showProgress = 0;
3917     }
3918 
3919     @Override
3920     public synchronized int summarize() {
3921       System.out.println(Integer.toString(errorCount) +
3922                          " inconsistencies detected.");
3923       if (errorCount == 0) {
3924         System.out.println("Status: OK");
3925         return 0;
3926       } else {
3927         System.out.println("Status: INCONSISTENT");
3928         return -1;
3929       }
3930     }
3931 
3932     @Override
3933     public ArrayList<ERROR_CODE> getErrorList() {
3934       return errorList;
3935     }
3936 
3937     @Override
3938     public synchronized void print(String message) {
3939       if (!getSUMMARY()) {
3940         System.out.println(message);
3941       }
3942     }
3943 
3944     private synchronized static boolean getSUMMARY() {
3945       return SUMMARY;
3946     }
3947 
3948     @Override
3949     public boolean tableHasErrors(TableInfo table) {
3950       return errorTables.contains(table);
3951     }
3952 
3953     @Override
3954     public void resetErrors() {
3955       errorCount = 0;
3956     }
3957 
3958     @Override
3959     public synchronized void detail(String message) {
3960       if (details) {
3961         System.out.println(message);
3962       }
3963       showProgress = 0;
3964     }
3965 
3966     @Override
3967     public synchronized void progress() {
3968       if (showProgress++ == progressThreshold) {
3969         if (!getSUMMARY()) {
3970           System.out.print(".");
3971         }
3972         showProgress = 0;
3973       }
3974     }
3975   }
3976 
3977   /**
3978    * Contact a region server and get all information from it
3979    */
3980   static class WorkItemRegion implements Callable<Void> {
3981     private HBaseFsck hbck;
3982     private ServerName rsinfo;
3983     private ErrorReporter errors;
3984     private HConnection connection;
3985 
3986     WorkItemRegion(HBaseFsck hbck, ServerName info,
3987                    ErrorReporter errors, HConnection connection) {
3988       this.hbck = hbck;
3989       this.rsinfo = info;
3990       this.errors = errors;
3991       this.connection = connection;
3992     }
3993 
3994     @Override
3995     public synchronized Void call() throws IOException {
3996       errors.progress();
3997       try {
3998         BlockingInterface server = connection.getAdmin(rsinfo);
3999 
4000         // list all online regions from this region server
4001         List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
4002         regions = filterRegions(regions);
4003 
4004         if (details) {
4005           errors.detail("RegionServer: " + rsinfo.getServerName() +
4006                            " number of regions: " + regions.size());
4007           for (HRegionInfo rinfo: regions) {
4008             errors.detail("  " + rinfo.getRegionNameAsString() +
4009                              " id: " + rinfo.getRegionId() +
4010                              " encoded_name: " + rinfo.getEncodedName() +
4011                              " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
4012                              " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
4013           }
4014         }
4015 
4016         // check to see if the existence of this region matches the region in META
4017         for (HRegionInfo r:regions) {
4018           HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
4019           hbi.addServer(r, rsinfo);
4020         }
4021       } catch (IOException e) {          // unable to connect to the region server.
4022         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
4023           " Unable to fetch region information. " + e);
4024         throw e;
4025       }
4026       return null;
4027     }
4028 
4029     private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
4030       List<HRegionInfo> ret = Lists.newArrayList();
4031       for (HRegionInfo hri : regions) {
4032         if (hri.isMetaTable() || (!hbck.checkMetaOnly
4033             && hbck.isTableIncluded(hri.getTable()))) {
4034           ret.add(hri);
4035         }
4036       }
4037       return ret;
4038     }
4039   }
4040 
4041   /**
4042    * Contact hdfs and get all information about specified table directory into
4043    * regioninfo list.
4044    */
4045   static class WorkItemHdfsDir implements Callable<Void> {
4046     private HBaseFsck hbck;
4047     private FileStatus tableDir;
4048     private ErrorReporter errors;
4049     private FileSystem fs;
4050 
4051     WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
4052                     FileStatus status) {
4053       this.hbck = hbck;
4054       this.fs = fs;
4055       this.tableDir = status;
4056       this.errors = errors;
4057     }
4058 
4059     @Override
4060     public synchronized Void call() throws IOException {
4061       try {
4062         // level 2: <HBASE_DIR>/<table>/*
4063         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
4064         for (FileStatus regionDir : regionDirs) {
4065           errors.progress();
4066           String encodedName = regionDir.getPath().getName();
4067           // ignore directories that aren't hexadecimal
4068           if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
4069             continue;
4070           }
4071 
4072           LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
4073           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
4074           HdfsEntry he = new HdfsEntry();
4075           synchronized (hbi) {
4076             if (hbi.getHdfsRegionDir() != null) {
4077               errors.print("Directory " + encodedName + " duplicate??" +
4078                            hbi.getHdfsRegionDir());
4079             }
4080 
4081             he.hdfsRegionDir = regionDir.getPath();
4082             he.hdfsRegionDirModTime = regionDir.getModificationTime();
4083             Path regioninfoFile = new Path(he.hdfsRegionDir, HRegionFileSystem.REGION_INFO_FILE);
4084             he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
4085             // we add to orphan list when we attempt to read .regioninfo
4086 
4087             // Set a flag if this region contains only edits
4088             // This is special case if a region is left after split
4089             he.hdfsOnlyEdits = true;
4090             FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
4091             Path ePath = WALSplitter.getRegionDirRecoveredEditsDir(regionDir.getPath());
4092             for (FileStatus subDir : subDirs) {
4093               errors.progress();
4094               String sdName = subDir.getPath().getName();
4095               if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
4096                 he.hdfsOnlyEdits = false;
4097                 break;
4098               }
4099             }
4100             hbi.hdfsEntry = he;
4101           }
4102         }
4103       } catch (IOException e) {
4104         // unable to connect to the region server.
4105         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
4106             + tableDir.getPath().getName()
4107             + " Unable to fetch region information. " + e);
4108         throw e;
4109       }
4110       return null;
4111     }
4112   }
4113 
4114   /**
4115    * Contact hdfs and get all information about specified table directory into
4116    * regioninfo list.
4117    */
4118   static class WorkItemHdfsRegionInfo implements Callable<Void> {
4119     private HbckInfo hbi;
4120     private HBaseFsck hbck;
4121     private ErrorReporter errors;
4122 
4123     WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
4124       this.hbi = hbi;
4125       this.hbck = hbck;
4126       this.errors = errors;
4127     }
4128 
4129     @Override
4130     public synchronized Void call() throws IOException {
4131       // only load entries that haven't been loaded yet.
4132       if (hbi.getHdfsHRI() == null) {
4133         try {
4134           errors.progress();
4135           hbck.loadHdfsRegioninfo(hbi);
4136         } catch (IOException ioe) {
4137           String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
4138               + hbi.getTableName() + " in hdfs dir "
4139               + hbi.getHdfsRegionDir()
4140               + "!  It may be an invalid format or version file.  Treating as "
4141               + "an orphaned regiondir.";
4142           errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
4143           try {
4144             hbck.debugLsr(hbi.getHdfsRegionDir());
4145           } catch (IOException ioe2) {
4146             LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
4147             throw ioe2;
4148           }
4149           hbck.orphanHdfsDirs.add(hbi);
4150           throw ioe;
4151         }
4152       }
4153       return null;
4154     }
4155   };
4156 
4157   /**
4158    * Display the full report from fsck. This displays all live and dead region
4159    * servers, and all known regions.
4160    */
4161   public static void setDisplayFullReport() {
4162     details = true;
4163   }
4164 
4165   /**
4166    * Set exclusive mode.
4167    */
4168   public static void setForceExclusive() {
4169     forceExclusive = true;
4170   }
4171 
4172   /**
4173    * Only one instance of hbck can modify HBase at a time.
4174    */
4175   public boolean isExclusive() {
4176     return fixAny || forceExclusive;
4177   }
4178 
4179   /**
4180    * Disable the load balancer.
4181    */
4182   public static void setDisableBalancer() {
4183     disableBalancer = true;
4184   }
4185 
4186   /**
4187    * The balancer should be disabled if we are modifying HBase.
4188    * It can be disabled if you want to prevent region movement from causing
4189    * false positives.
4190    */
4191   public boolean shouldDisableBalancer() {
4192     return fixAny || disableBalancer;
4193   }
4194 
4195   /**
4196    * Set summary mode.
4197    * Print only summary of the tables and status (OK or INCONSISTENT)
4198    */
4199   synchronized static void setSummary() {
4200     SUMMARY = true;
4201   }
4202 
4203   /**
4204    * Set hbase:meta check mode.
4205    * Print only info about hbase:meta table deployment/state
4206    */
4207   void setCheckMetaOnly() {
4208     checkMetaOnly = true;
4209   }
4210 
4211   /**
4212    * Set region boundaries check mode.
4213    */
4214   void setRegionBoundariesCheck() {
4215     checkRegionBoundaries = true;
4216   }
4217 
4218   /**
4219    * Set table locks fix mode.
4220    * Delete table locks held for a long time
4221    */
4222   public void setFixTableLocks(boolean shouldFix) {
4223     fixTableLocks = shouldFix;
4224     fixAny |= shouldFix;
4225   }
4226 
4227   /**
4228    * Set orphaned table ZNodes fix mode.
4229    * Set the table state to disable in the orphaned table ZNode.
4230    */
4231   public void setFixTableZNodes(boolean shouldFix) {
4232     fixTableZNodes = shouldFix;
4233     fixAny |= shouldFix;
4234   }
4235 
4236   /**
4237    * Check if we should rerun fsck again. This checks if we've tried to
4238    * fix something and we should rerun fsck tool again.
4239    * Display the full report from fsck. This displays all live and dead
4240    * region servers, and all known regions.
4241    */
4242   void setShouldRerun() {
4243     rerun = true;
4244   }
4245 
4246   boolean shouldRerun() {
4247     return rerun;
4248   }
4249 
4250   /**
4251    * Fix inconsistencies found by fsck. This should try to fix errors (if any)
4252    * found by fsck utility.
4253    */
4254   public void setFixAssignments(boolean shouldFix) {
4255     fixAssignments = shouldFix;
4256     fixAny |= shouldFix;
4257   }
4258 
4259   boolean shouldFixAssignments() {
4260     return fixAssignments;
4261   }
4262 
4263   public void setFixMeta(boolean shouldFix) {
4264     fixMeta = shouldFix;
4265     fixAny |= shouldFix;
4266   }
4267 
4268   boolean shouldFixMeta() {
4269     return fixMeta;
4270   }
4271 
4272   public void setFixEmptyMetaCells(boolean shouldFix) {
4273     fixEmptyMetaCells = shouldFix;
4274     fixAny |= shouldFix;
4275   }
4276 
4277   boolean shouldFixEmptyMetaCells() {
4278     return fixEmptyMetaCells;
4279   }
4280 
4281   public void setCheckHdfs(boolean checking) {
4282     checkHdfs = checking;
4283   }
4284 
4285   boolean shouldCheckHdfs() {
4286     return checkHdfs;
4287   }
4288 
4289   public void setFixHdfsHoles(boolean shouldFix) {
4290     fixHdfsHoles = shouldFix;
4291     fixAny |= shouldFix;
4292   }
4293 
4294   boolean shouldFixHdfsHoles() {
4295     return fixHdfsHoles;
4296   }
4297 
4298   public void setFixTableOrphans(boolean shouldFix) {
4299     fixTableOrphans = shouldFix;
4300     fixAny |= shouldFix;
4301   }
4302 
4303   boolean shouldFixTableOrphans() {
4304     return fixTableOrphans;
4305   }
4306 
4307   public void setFixHdfsOverlaps(boolean shouldFix) {
4308     fixHdfsOverlaps = shouldFix;
4309     fixAny |= shouldFix;
4310   }
4311 
4312   boolean shouldFixHdfsOverlaps() {
4313     return fixHdfsOverlaps;
4314   }
4315 
4316   public void setFixHdfsOrphans(boolean shouldFix) {
4317     fixHdfsOrphans = shouldFix;
4318     fixAny |= shouldFix;
4319   }
4320 
4321   boolean shouldFixHdfsOrphans() {
4322     return fixHdfsOrphans;
4323   }
4324 
4325   public void setFixVersionFile(boolean shouldFix) {
4326     fixVersionFile = shouldFix;
4327     fixAny |= shouldFix;
4328   }
4329 
4330   public boolean shouldFixVersionFile() {
4331     return fixVersionFile;
4332   }
4333 
4334   public void setSidelineBigOverlaps(boolean sbo) {
4335     this.sidelineBigOverlaps = sbo;
4336   }
4337 
4338   public boolean shouldSidelineBigOverlaps() {
4339     return sidelineBigOverlaps;
4340   }
4341 
4342   public void setFixSplitParents(boolean shouldFix) {
4343     fixSplitParents = shouldFix;
4344     fixAny |= shouldFix;
4345   }
4346 
4347   boolean shouldFixSplitParents() {
4348     return fixSplitParents;
4349   }
4350 
4351   public void setFixReferenceFiles(boolean shouldFix) {
4352     fixReferenceFiles = shouldFix;
4353     fixAny |= shouldFix;
4354   }
4355 
4356   boolean shouldFixReferenceFiles() {
4357     return fixReferenceFiles;
4358   }
4359 
4360   public boolean shouldIgnorePreCheckPermission() {
4361     return !fixAny || ignorePreCheckPermission;
4362   }
4363 
4364   public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
4365     this.ignorePreCheckPermission = ignorePreCheckPermission;
4366   }
4367 
4368   /**
4369    * @param mm maximum number of regions to merge into a single region.
4370    */
4371   public void setMaxMerge(int mm) {
4372     this.maxMerge = mm;
4373   }
4374 
4375   public int getMaxMerge() {
4376     return maxMerge;
4377   }
4378 
4379   public void setMaxOverlapsToSideline(int mo) {
4380     this.maxOverlapsToSideline = mo;
4381   }
4382 
4383   public int getMaxOverlapsToSideline() {
4384     return maxOverlapsToSideline;
4385   }
4386 
4387   /**
4388    * Only check/fix tables specified by the list,
4389    * Empty list means all tables are included.
4390    */
4391   boolean isTableIncluded(TableName table) {
4392     return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
4393   }
4394 
4395   public void includeTable(TableName table) {
4396     tablesIncluded.add(table);
4397   }
4398 
4399   Set<TableName> getIncludedTables() {
4400     return new HashSet<TableName>(tablesIncluded);
4401   }
4402 
4403   /**
4404    * We are interested in only those tables that have not changed their state in
4405    * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
4406    * @param seconds - the time in seconds
4407    */
4408   public void setTimeLag(long seconds) {
4409     timelag = seconds * 1000; // convert to milliseconds
4410   }
4411 
4412   /**
4413    *
4414    * @param sidelineDir - HDFS path to sideline data
4415    */
4416   public void setSidelineDir(String sidelineDir) {
4417     this.sidelineDir = new Path(sidelineDir);
4418   }
4419 
4420   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
4421     return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
4422   }
4423 
4424   public HFileCorruptionChecker getHFilecorruptionChecker() {
4425     return hfcc;
4426   }
4427 
4428   public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
4429     this.hfcc = hfcc;
4430   }
4431 
4432   public void setRetCode(int code) {
4433     this.retcode = code;
4434   }
4435 
4436   public int getRetCode() {
4437     return retcode;
4438   }
4439 
4440   protected HBaseFsck printUsageAndExit() {
4441     StringWriter sw = new StringWriter(2048);
4442     PrintWriter out = new PrintWriter(sw);
4443     out.println("Usage: fsck [opts] {only tables}");
4444     out.println(" where [opts] are:");
4445     out.println("   -help Display help options (this)");
4446     out.println("   -details Display full report of all regions.");
4447     out.println("   -timelag <timeInSeconds>  Process only regions that " +
4448                        " have not experienced any metadata updates in the last " +
4449                        " <timeInSeconds> seconds.");
4450     out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
4451         " before checking if the fix worked if run with -fix");
4452     out.println("   -summary Print only summary of the tables and status.");
4453     out.println("   -metaonly Only check the state of the hbase:meta table.");
4454     out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
4455     out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
4456     out.println("   -exclusive Abort if another hbck is exclusive or fixing.");
4457     out.println("   -disableBalancer Disable the load balancer.");
4458 
4459     out.println("");
4460     out.println("  Metadata Repair options: (expert features, use with caution!)");
4461     out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
4462     out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
4463     out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
4464     out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
4465         + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
4466     out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
4467     out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
4468     out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
4469     out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
4470     out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
4471     out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
4472     out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
4473     out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
4474     out.println("   -fixSplitParents  Try to force offline split parents to be online.");
4475     out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
4476     out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
4477     out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
4478         + " (empty REGIONINFO_QUALIFIER rows)");
4479 
4480     out.println("");
4481     out.println("  Datafile Repair options: (expert features, use with caution!)");
4482     out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
4483     out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
4484 
4485     out.println("");
4486     out.println("  Metadata Repair shortcuts");
4487     out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
4488         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps " +
4489         "-fixReferenceFiles -fixTableLocks -fixOrphanedTableZnodes");
4490     out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
4491 
4492     out.println("");
4493     out.println("  Table lock options");
4494     out.println("   -fixTableLocks    Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
4495 
4496     out.println("");
4497     out.println("  Table Znode options");
4498     out.println("   -fixOrphanedTableZnodes    Set table state in ZNode to disabled if table does not exists");
4499 
4500     out.flush();
4501     errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
4502 
4503     setRetCode(-2);
4504     return this;
4505   }
4506 
4507   /**
4508    * Main program
4509    *
4510    * @param args
4511    * @throws Exception
4512    */
4513   public static void main(String[] args) throws Exception {
4514     // create a fsck object
4515     Configuration conf = HBaseConfiguration.create();
4516     Path hbasedir = FSUtils.getRootDir(conf);
4517     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
4518     FSUtils.setFsDefault(conf, new Path(defaultFs));
4519     int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
4520     System.exit(ret);
4521   }
4522 
4523   /**
4524    * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
4525    */
4526   static class HBaseFsckTool extends Configured implements Tool {
4527     HBaseFsckTool(Configuration conf) { super(conf); }
4528     @Override
4529     public int run(String[] args) throws Exception {
4530       HBaseFsck hbck = new HBaseFsck(getConf());
4531       hbck.exec(hbck.executor, args);
4532       hbck.close();
4533       return hbck.getRetCode();
4534     }
4535   };
4536 
4537 
4538   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
4539     ServiceException, InterruptedException {
4540     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
4541 
4542     boolean checkCorruptHFiles = false;
4543     boolean sidelineCorruptHFiles = false;
4544 
4545     // Process command-line args.
4546     for (int i = 0; i < args.length; i++) {
4547       String cmd = args[i];
4548       if (cmd.equals("-help") || cmd.equals("-h")) {
4549         return printUsageAndExit();
4550       } else if (cmd.equals("-details")) {
4551         setDisplayFullReport();
4552       } else if (cmd.equals("-exclusive")) {
4553         setForceExclusive();
4554       } else if (cmd.equals("-disableBalancer")) {
4555         setDisableBalancer();
4556       } else if (cmd.equals("-timelag")) {
4557         if (i == args.length - 1) {
4558           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4559           return printUsageAndExit();
4560         }
4561         try {
4562           long timelag = Long.parseLong(args[i+1]);
4563           setTimeLag(timelag);
4564         } catch (NumberFormatException e) {
4565           errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4566           return printUsageAndExit();
4567         }
4568         i++;
4569       } else if (cmd.equals("-sleepBeforeRerun")) {
4570         if (i == args.length - 1) {
4571           errors.reportError(ERROR_CODE.WRONG_USAGE,
4572             "HBaseFsck: -sleepBeforeRerun needs a value.");
4573           return printUsageAndExit();
4574         }
4575         try {
4576           sleepBeforeRerun = Long.parseLong(args[i+1]);
4577         } catch (NumberFormatException e) {
4578           errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4579           return printUsageAndExit();
4580         }
4581         i++;
4582       } else if (cmd.equals("-sidelineDir")) {
4583         if (i == args.length - 1) {
4584           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4585           return printUsageAndExit();
4586         }
4587         i++;
4588         setSidelineDir(args[i]);
4589       } else if (cmd.equals("-fix")) {
4590         errors.reportError(ERROR_CODE.WRONG_USAGE,
4591           "This option is deprecated, please use  -fixAssignments instead.");
4592         setFixAssignments(true);
4593       } else if (cmd.equals("-fixAssignments")) {
4594         setFixAssignments(true);
4595       } else if (cmd.equals("-fixMeta")) {
4596         setFixMeta(true);
4597       } else if (cmd.equals("-noHdfsChecking")) {
4598         setCheckHdfs(false);
4599       } else if (cmd.equals("-fixHdfsHoles")) {
4600         setFixHdfsHoles(true);
4601       } else if (cmd.equals("-fixHdfsOrphans")) {
4602         setFixHdfsOrphans(true);
4603       } else if (cmd.equals("-fixTableOrphans")) {
4604         setFixTableOrphans(true);
4605       } else if (cmd.equals("-fixHdfsOverlaps")) {
4606         setFixHdfsOverlaps(true);
4607       } else if (cmd.equals("-fixVersionFile")) {
4608         setFixVersionFile(true);
4609       } else if (cmd.equals("-sidelineBigOverlaps")) {
4610         setSidelineBigOverlaps(true);
4611       } else if (cmd.equals("-fixSplitParents")) {
4612         setFixSplitParents(true);
4613       } else if (cmd.equals("-ignorePreCheckPermission")) {
4614         setIgnorePreCheckPermission(true);
4615       } else if (cmd.equals("-checkCorruptHFiles")) {
4616         checkCorruptHFiles = true;
4617       } else if (cmd.equals("-sidelineCorruptHFiles")) {
4618         sidelineCorruptHFiles = true;
4619       } else if (cmd.equals("-fixReferenceFiles")) {
4620         setFixReferenceFiles(true);
4621       } else if (cmd.equals("-fixEmptyMetaCells")) {
4622         setFixEmptyMetaCells(true);
4623       } else if (cmd.equals("-repair")) {
4624         // this attempts to merge overlapping hdfs regions, needs testing
4625         // under load
4626         setFixHdfsHoles(true);
4627         setFixHdfsOrphans(true);
4628         setFixMeta(true);
4629         setFixAssignments(true);
4630         setFixHdfsOverlaps(true);
4631         setFixVersionFile(true);
4632         setSidelineBigOverlaps(true);
4633         setFixSplitParents(false);
4634         setCheckHdfs(true);
4635         setFixReferenceFiles(true);
4636         setFixTableLocks(true);
4637         setFixTableZNodes(true);
4638       } else if (cmd.equals("-repairHoles")) {
4639         // this will make all missing hdfs regions available but may lose data
4640         setFixHdfsHoles(true);
4641         setFixHdfsOrphans(false);
4642         setFixMeta(true);
4643         setFixAssignments(true);
4644         setFixHdfsOverlaps(false);
4645         setSidelineBigOverlaps(false);
4646         setFixSplitParents(false);
4647         setCheckHdfs(true);
4648       } else if (cmd.equals("-maxOverlapsToSideline")) {
4649         if (i == args.length - 1) {
4650           errors.reportError(ERROR_CODE.WRONG_USAGE,
4651             "-maxOverlapsToSideline needs a numeric value argument.");
4652           return printUsageAndExit();
4653         }
4654         try {
4655           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4656           setMaxOverlapsToSideline(maxOverlapsToSideline);
4657         } catch (NumberFormatException e) {
4658           errors.reportError(ERROR_CODE.WRONG_USAGE,
4659             "-maxOverlapsToSideline needs a numeric value argument.");
4660           return printUsageAndExit();
4661         }
4662         i++;
4663       } else if (cmd.equals("-maxMerge")) {
4664         if (i == args.length - 1) {
4665           errors.reportError(ERROR_CODE.WRONG_USAGE,
4666             "-maxMerge needs a numeric value argument.");
4667           return printUsageAndExit();
4668         }
4669         try {
4670           int maxMerge = Integer.parseInt(args[i+1]);
4671           setMaxMerge(maxMerge);
4672         } catch (NumberFormatException e) {
4673           errors.reportError(ERROR_CODE.WRONG_USAGE,
4674             "-maxMerge needs a numeric value argument.");
4675           return printUsageAndExit();
4676         }
4677         i++;
4678       } else if (cmd.equals("-summary")) {
4679         setSummary();
4680       } else if (cmd.equals("-metaonly")) {
4681         setCheckMetaOnly();
4682       } else if (cmd.equals("-boundaries")) {
4683         setRegionBoundariesCheck();
4684       } else if (cmd.equals("-fixTableLocks")) {
4685         setFixTableLocks(true);
4686       } else if (cmd.equals("-fixOrphanedTableZnodes")) {
4687         setFixTableZNodes(true);
4688       } else if (cmd.startsWith("-")) {
4689         errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4690         return printUsageAndExit();
4691       } else {
4692         includeTable(TableName.valueOf(cmd));
4693         errors.print("Allow checking/fixes for table: " + cmd);
4694       }
4695     }
4696 
4697     errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
4698 
4699     // pre-check current user has FS write permission or not
4700     try {
4701       preCheckPermission();
4702     } catch (AccessDeniedException ace) {
4703       Runtime.getRuntime().exit(-1);
4704     } catch (IOException ioe) {
4705       Runtime.getRuntime().exit(-1);
4706     }
4707 
4708     // do the real work of hbck
4709     connect();
4710 
4711     try {
4712       // if corrupt file mode is on, first fix them since they may be opened later
4713       if (checkCorruptHFiles || sidelineCorruptHFiles) {
4714         LOG.info("Checking all hfiles for corruption");
4715         HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4716         setHFileCorruptionChecker(hfcc); // so we can get result
4717         Collection<TableName> tables = getIncludedTables();
4718         Collection<Path> tableDirs = new ArrayList<Path>();
4719         Path rootdir = FSUtils.getRootDir(getConf());
4720         if (tables.size() > 0) {
4721           for (TableName t : tables) {
4722             tableDirs.add(FSUtils.getTableDir(rootdir, t));
4723           }
4724         } else {
4725           tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4726         }
4727         hfcc.checkTables(tableDirs);
4728         hfcc.report(errors);
4729       }
4730 
4731       // check and fix table integrity, region consistency.
4732       int code = onlineHbck();
4733       setRetCode(code);
4734       // If we have changed the HBase state it is better to run hbck again
4735       // to see if we haven't broken something else in the process.
4736       // We run it only once more because otherwise we can easily fall into
4737       // an infinite loop.
4738       if (shouldRerun()) {
4739         try {
4740           LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4741           Thread.sleep(sleepBeforeRerun);
4742         } catch (InterruptedException ie) {
4743           LOG.warn("Interrupted while sleeping");
4744           return this;
4745         }
4746         // Just report
4747         setFixAssignments(false);
4748         setFixMeta(false);
4749         setFixHdfsHoles(false);
4750         setFixHdfsOverlaps(false);
4751         setFixVersionFile(false);
4752         setFixTableOrphans(false);
4753         errors.resetErrors();
4754         code = onlineHbck();
4755         setRetCode(code);
4756       }
4757     } finally {
4758       IOUtils.closeQuietly(this);
4759     }
4760     return this;
4761   }
4762 
4763   /**
4764    * ls -r for debugging purposes
4765    */
4766   void debugLsr(Path p) throws IOException {
4767     debugLsr(getConf(), p, errors);
4768   }
4769 
4770   /**
4771    * ls -r for debugging purposes
4772    */
4773   public static void debugLsr(Configuration conf,
4774       Path p) throws IOException {
4775     debugLsr(conf, p, new PrintingErrorReporter());
4776   }
4777 
4778   /**
4779    * ls -r for debugging purposes
4780    */
4781   public static void debugLsr(Configuration conf,
4782       Path p, ErrorReporter errors) throws IOException {
4783     if (!LOG.isDebugEnabled() || p == null) {
4784       return;
4785     }
4786     FileSystem fs = p.getFileSystem(conf);
4787 
4788     if (!fs.exists(p)) {
4789       // nothing
4790       return;
4791     }
4792     errors.print(p.toString());
4793 
4794     if (fs.isFile(p)) {
4795       return;
4796     }
4797 
4798     if (fs.getFileStatus(p).isDirectory()) {
4799       FileStatus[] fss= fs.listStatus(p);
4800       for (FileStatus status : fss) {
4801         debugLsr(conf, status.getPath(), errors);
4802       }
4803     }
4804   }
4805 }