View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.Closeable;
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.io.InterruptedIOException;
24  import java.io.PrintWriter;
25  import java.io.StringWriter;
26  import java.net.InetAddress;
27  import java.net.URI;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collection;
31  import java.util.Collections;
32  import java.util.Comparator;
33  import java.util.HashMap;
34  import java.util.HashSet;
35  import java.util.Iterator;
36  import java.util.List;
37  import java.util.Map;
38  import java.util.Map.Entry;
39  import java.util.Set;
40  import java.util.SortedMap;
41  import java.util.SortedSet;
42  import java.util.TreeMap;
43  import java.util.TreeSet;
44  import java.util.concurrent.Callable;
45  import java.util.concurrent.ConcurrentSkipListMap;
46  import java.util.concurrent.ExecutionException;
47  import java.util.concurrent.ExecutorService;
48  import java.util.concurrent.Executors;
49  import java.util.concurrent.Future;
50  import java.util.concurrent.FutureTask;
51  import java.util.concurrent.ScheduledThreadPoolExecutor;
52  import java.util.concurrent.TimeUnit;
53  import java.util.concurrent.TimeoutException;
54  import java.util.concurrent.atomic.AtomicBoolean;
55  import java.util.concurrent.atomic.AtomicInteger;
56  
57  import org.apache.commons.io.IOUtils;
58  import org.apache.commons.lang.StringUtils;
59  import org.apache.commons.logging.Log;
60  import org.apache.commons.logging.LogFactory;
61  import org.apache.hadoop.conf.Configuration;
62  import org.apache.hadoop.conf.Configured;
63  import org.apache.hadoop.fs.FSDataOutputStream;
64  import org.apache.hadoop.fs.FileStatus;
65  import org.apache.hadoop.fs.FileSystem;
66  import org.apache.hadoop.fs.Path;
67  import org.apache.hadoop.fs.permission.FsAction;
68  import org.apache.hadoop.fs.permission.FsPermission;
69  import org.apache.hadoop.hbase.Abortable;
70  import org.apache.hadoop.hbase.Cell;
71  import org.apache.hadoop.hbase.ClusterStatus;
72  import org.apache.hadoop.hbase.CoordinatedStateException;
73  import org.apache.hadoop.hbase.HBaseConfiguration;
74  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
75  import org.apache.hadoop.hbase.HColumnDescriptor;
76  import org.apache.hadoop.hbase.HConstants;
77  import org.apache.hadoop.hbase.HRegionInfo;
78  import org.apache.hadoop.hbase.HRegionLocation;
79  import org.apache.hadoop.hbase.HTableDescriptor;
80  import org.apache.hadoop.hbase.KeyValue;
81  import org.apache.hadoop.hbase.MasterNotRunningException;
82  import org.apache.hadoop.hbase.MetaTableAccessor;
83  import org.apache.hadoop.hbase.RegionLocations;
84  import org.apache.hadoop.hbase.ServerName;
85  import org.apache.hadoop.hbase.TableName;
86  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
87  import org.apache.hadoop.hbase.classification.InterfaceAudience;
88  import org.apache.hadoop.hbase.classification.InterfaceStability;
89  import org.apache.hadoop.hbase.client.Admin;
90  import org.apache.hadoop.hbase.client.ClusterConnection;
91  import org.apache.hadoop.hbase.client.ConnectionFactory;
92  import org.apache.hadoop.hbase.client.Delete;
93  import org.apache.hadoop.hbase.client.Get;
94  import org.apache.hadoop.hbase.client.HBaseAdmin;
95  import org.apache.hadoop.hbase.client.HConnectable;
96  import org.apache.hadoop.hbase.client.HConnection;
97  import org.apache.hadoop.hbase.client.HConnectionManager;
98  import org.apache.hadoop.hbase.client.MetaScanner;
99  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
100 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
101 import org.apache.hadoop.hbase.client.Put;
102 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
103 import org.apache.hadoop.hbase.client.Result;
104 import org.apache.hadoop.hbase.client.RowMutations;
105 import org.apache.hadoop.hbase.client.Table;
106 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
107 import org.apache.hadoop.hbase.io.hfile.HFile;
108 import org.apache.hadoop.hbase.master.MasterFileSystem;
109 import org.apache.hadoop.hbase.master.RegionState;
110 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
111 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
112 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
113 import org.apache.hadoop.hbase.regionserver.HRegion;
114 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
115 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
116 import org.apache.hadoop.hbase.security.AccessDeniedException;
117 import org.apache.hadoop.hbase.security.UserProvider;
118 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
119 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
120 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
121 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
122 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
123 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
124 import org.apache.hadoop.hbase.wal.WALSplitter;
125 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
126 import org.apache.hadoop.hbase.zookeeper.ZKTableStateClientSideReader;
127 import org.apache.hadoop.hbase.zookeeper.ZKTableStateManager;
128 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
129 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
130 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
131 import org.apache.hadoop.ipc.RemoteException;
132 import org.apache.hadoop.security.UserGroupInformation;
133 import org.apache.hadoop.util.ReflectionUtils;
134 import org.apache.hadoop.util.Tool;
135 import org.apache.hadoop.util.ToolRunner;
136 import org.apache.zookeeper.KeeperException;
137 
138 import com.google.common.base.Joiner;
139 import com.google.common.base.Preconditions;
140 import com.google.common.collect.ImmutableList;
141 import com.google.common.collect.Lists;
142 import com.google.common.collect.Multimap;
143 import com.google.common.collect.Ordering;
144 import com.google.common.collect.TreeMultimap;
145 import com.google.protobuf.ServiceException;
146 
147 /**
148  * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
149  * table integrity problems in a corrupted HBase.
150  * <p>
151  * Region consistency checks verify that hbase:meta, region deployment on region
152  * servers and the state of data in HDFS (.regioninfo files) all are in
153  * accordance.
154  * <p>
155  * Table integrity checks verify that all possible row keys resolve to exactly
156  * one region of a table.  This means there are no individual degenerate
157  * or backwards regions; no holes between regions; and that there are no
158  * overlapping regions.
159  * <p>
160  * The general repair strategy works in two phases:
161  * <ol>
162  * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
163  * <li> Repair Region Consistency with hbase:meta and assignments
164  * </ol>
165  * <p>
166  * For table integrity repairs, the tables' region directories are scanned
167  * for .regioninfo files.  Each table's integrity is then verified.  If there
168  * are any orphan regions (regions with no .regioninfo files) or holes, new
169  * regions are fabricated.  Backwards regions are sidelined as well as empty
170  * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
171  * a new region is created and all data is merged into the new region.
172  * <p>
173  * Table integrity repairs deal solely with HDFS and could potentially be done
174  * offline -- the hbase region servers or master do not need to be running.
175  * This phase can eventually be used to completely reconstruct the hbase:meta table in
176  * an offline fashion.
177  * <p>
178  * Region consistency requires three conditions -- 1) valid .regioninfo file
179  * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
180  * and 3) a region is deployed only at the regionserver that was assigned to
181  * with proper state in the master.
182  * <p>
183  * Region consistency repairs require hbase to be online so that hbck can
184  * contact the HBase master and region servers.  The hbck#connect() method must
185  * first be called successfully.  Much of the region consistency information
186  * is transient and less risky to repair.
187  * <p>
188  * If hbck is run from the command line, there are a handful of arguments that
189  * can be used to limit the kinds of repairs hbck will do.  See the code in
190  * {@link #printUsageAndExit()} for more details.
191  */
192 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
193 @InterfaceStability.Evolving
194 public class HBaseFsck extends Configured implements Closeable {
195   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
196   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
197   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
198   private static boolean rsSupportsOffline = true;
199   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
200   private static final int DEFAULT_MAX_MERGE = 5;
201   private static final String TO_BE_LOADED = "to_be_loaded";
202   private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
203   private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5;
204   private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds
205   private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds
206   // We have to set the timeout value > HdfsConstants.LEASE_SOFTLIMIT_PERIOD.
207   // In HADOOP-2.6 and later, the Namenode proxy now created with custom RetryPolicy for
208   // AlreadyBeingCreatedException which is implies timeout on this operations up to
209   // HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds).
210   private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds
211 
212   /**********************
213    * Internal resources
214    **********************/
215   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
216   private ClusterStatus status;
217   private ClusterConnection connection;
218   private Admin admin;
219   private Table meta;
220   // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
221   protected ExecutorService executor;
222   private long startMillis = EnvironmentEdgeManager.currentTime();
223   private HFileCorruptionChecker hfcc;
224   private int retcode = 0;
225   private Path HBCK_LOCK_PATH;
226   private FSDataOutputStream hbckOutFd;
227   // This lock is to prevent cleanup of balancer resources twice between
228   // ShutdownHook and the main code. We cleanup only if the connect() is
229   // successful
230   private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
231 
232   /***********
233    * Options
234    ***********/
235   private static boolean details = false; // do we display the full report
236   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
237   private static boolean forceExclusive = false; // only this hbck can modify HBase
238   private static boolean disableBalancer = false; // disable load balancer to keep regions stable
239   private boolean fixAssignments = false; // fix assignment errors?
240   private boolean fixMeta = false; // fix meta errors?
241   private boolean checkHdfs = true; // load and check fs consistency?
242   private boolean fixHdfsHoles = false; // fix fs holes?
243   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
244   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
245   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
246   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
247   private boolean fixSplitParents = false; // fix lingering split parents
248   private boolean removeParents = false; // remove split parents
249   private boolean fixReferenceFiles = false; // fix lingering reference store file
250   private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
251   private boolean fixTableLocks = false; // fix table locks which are expired
252   private boolean fixTableZNodes = false; // fix table Znodes which are orphaned
253   private boolean fixAny = false; // Set to true if any of the fix is required.
254 
255   // limit checking/fixes to listed tables, if empty attempt to check/fix all
256   // hbase:meta are always checked
257   private Set<TableName> tablesIncluded = new HashSet<TableName>();
258   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
259   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
260   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
261   private Path sidelineDir = null;
262 
263   private boolean rerun = false; // if we tried to fix something, rerun hbck
264   private static boolean SUMMARY = false; // if we want to print less output
265   private boolean checkMetaOnly = false;
266   private boolean checkRegionBoundaries = false;
267   private boolean ignorePreCheckPermission = false; // if pre-check permission
268 
269   /*********
270    * State
271    *********/
272   final private ErrorReporter errors;
273   int fixes = 0;
274 
275   /**
276    * This map contains the state of all hbck items.  It maps from encoded region
277    * name to HbckInfo structure.  The information contained in HbckInfo is used
278    * to detect and correct consistency (hdfs/meta/deployment) problems.
279    */
280   private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
281   private TreeSet<TableName> disabledTables =
282     new TreeSet<TableName>();
283   // Empty regioninfo qualifiers in hbase:meta
284   private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
285 
286   /**
287    * This map from Tablename -> TableInfo contains the structures necessary to
288    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
289    * to prevent dupes.
290    *
291    * If tablesIncluded is empty, this map contains all tables.
292    * Otherwise, it contains only meta tables and tables in tablesIncluded,
293    * unless checkMetaOnly is specified, in which case, it contains only
294    * the meta table
295    */
296   private SortedMap<TableName, TableInfo> tablesInfo =
297       new ConcurrentSkipListMap<TableName, TableInfo>();
298 
299   /**
300    * When initially looking at HDFS, we attempt to find any orphaned data.
301    */
302   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
303 
304   private Map<TableName, Set<String>> orphanTableDirs =
305       new HashMap<TableName, Set<String>>();
306 
307   private Map<TableName, Set<String>> skippedRegions = new HashMap<TableName, Set<String>>();
308 
309   /**
310    * List of orphaned table ZNodes
311    */
312   private Set<TableName> orphanedTableZNodes = new HashSet<TableName>();
313   private final RetryCounterFactory lockFileRetryCounterFactory;
314   
315 
316   /**
317    * Constructor
318    *
319    * @param conf Configuration object
320    * @throws MasterNotRunningException if the master is not running
321    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
322    */
323   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
324       ZooKeeperConnectionException, IOException, ClassNotFoundException {
325     this(conf, createThreadPool(conf));
326   }
327 
328   private static ExecutorService createThreadPool(Configuration conf) {
329     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
330     return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
331   }
332 
333   /**
334    * Constructor
335    *
336    * @param conf
337    *          Configuration object
338    * @throws MasterNotRunningException
339    *           if the master is not running
340    * @throws ZooKeeperConnectionException
341    *           if unable to connect to ZooKeeper
342    */
343   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
344       ZooKeeperConnectionException, IOException, ClassNotFoundException {
345     super(conf);
346     errors = getErrorReporter(getConf());
347     this.executor = exec;
348     lockFileRetryCounterFactory = new RetryCounterFactory(
349       getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),
350       getConf().getInt(
351         "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),
352       getConf().getInt(
353         "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));
354   }
355 
356   private class FileLockCallable implements Callable<FSDataOutputStream> {
357     RetryCounter retryCounter;
358 
359     public FileLockCallable(RetryCounter retryCounter) {
360       this.retryCounter = retryCounter;
361     }
362     @Override
363     public FSDataOutputStream call() throws IOException {
364       try {
365         FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
366         FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
367             HConstants.DATA_FILE_UMASK_KEY);
368         Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
369         fs.mkdirs(tmpDir);
370         HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
371         final FSDataOutputStream out = createFileWithRetries(fs, HBCK_LOCK_PATH, defaultPerms);
372         out.writeBytes(InetAddress.getLocalHost().toString());
373         out.flush();
374         return out;
375       } catch(RemoteException e) {
376         if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
377           return null;
378         } else {
379           throw e;
380         }
381       }
382     }
383 
384     private FSDataOutputStream createFileWithRetries(final FileSystem fs,
385         final Path hbckLockFilePath, final FsPermission defaultPerms)
386         throws IOException {
387 
388       IOException exception = null;
389       do {
390         try {
391           return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false);
392         } catch (IOException ioe) {
393           LOG.info("Failed to create lock file " + hbckLockFilePath.getName()
394               + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of "
395               + retryCounter.getMaxAttempts());
396           LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(), 
397               ioe);
398           try {
399             exception = ioe;
400             retryCounter.sleepUntilNextRetry();
401           } catch (InterruptedException ie) {
402             throw (InterruptedIOException) new InterruptedIOException(
403                 "Can't create lock file " + hbckLockFilePath.getName())
404             .initCause(ie);
405           }
406         }
407       } while (retryCounter.shouldRetry());
408 
409       throw exception;
410     }
411   }
412 
413   /**
414    * This method maintains a lock using a file. If the creation fails we return null
415    *
416    * @return FSDataOutputStream object corresponding to the newly opened lock file
417    * @throws IOException
418    */
419   private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
420     RetryCounter retryCounter = lockFileRetryCounterFactory.create();
421     FileLockCallable callable = new FileLockCallable(retryCounter);
422     ExecutorService executor = Executors.newFixedThreadPool(1);
423     FutureTask<FSDataOutputStream> futureTask = new FutureTask<FSDataOutputStream>(callable);
424     executor.execute(futureTask);
425     final int timeoutInSeconds = getConf().getInt(
426       "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT);
427     FSDataOutputStream stream = null;
428     try {
429       stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);
430     } catch (ExecutionException ee) {
431       LOG.warn("Encountered exception when opening lock file", ee);
432     } catch (InterruptedException ie) {
433       LOG.warn("Interrupted when opening lock file", ie);
434       Thread.currentThread().interrupt();
435     } catch (TimeoutException exception) {
436       // took too long to obtain lock
437       LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");
438       futureTask.cancel(true);
439     } finally {
440       executor.shutdownNow();
441     }
442     return stream;
443   }
444 
445   private void unlockHbck() {
446     if (isExclusive() && hbckLockCleanup.compareAndSet(true, false)) {
447       RetryCounter retryCounter = lockFileRetryCounterFactory.create();
448       do {
449         try {
450           IOUtils.closeQuietly(hbckOutFd);
451           FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()),
452               HBCK_LOCK_PATH, true);
453           LOG.info("Finishing hbck");
454           return;
455         } catch (IOException ioe) {
456           LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try="
457               + (retryCounter.getAttemptTimes() + 1) + " of "
458               + retryCounter.getMaxAttempts());
459           LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe);
460           try {
461             retryCounter.sleepUntilNextRetry();
462           } catch (InterruptedException ie) {
463             Thread.currentThread().interrupt();
464             LOG.warn("Interrupted while deleting lock file" +
465                 HBCK_LOCK_PATH);
466             return;
467           }
468         }
469       } while (retryCounter.shouldRetry());
470     }
471   }
472 
473   /**
474    * To repair region consistency, one must call connect() in order to repair
475    * online state.
476    */
477   public void connect() throws IOException {
478 
479     if (isExclusive()) {
480       // Grab the lock
481       hbckOutFd = checkAndMarkRunningHbck();
482       if (hbckOutFd == null) {
483         setRetCode(-1);
484         LOG.error("Another instance of hbck is fixing HBase, exiting this instance. " +
485             "[If you are sure no other instance is running, delete the lock file " +
486             HBCK_LOCK_PATH + " and rerun the tool]");
487         throw new IOException("Duplicate hbck - Abort");
488       }
489 
490       // Make sure to cleanup the lock
491       hbckLockCleanup.set(true);
492     }
493 
494 
495     // Add a shutdown hook to this thread, in case user tries to
496     // kill the hbck with a ctrl-c, we want to cleanup the lock so that
497     // it is available for further calls
498     Runtime.getRuntime().addShutdownHook(new Thread() {
499       @Override
500       public void run() {
501         IOUtils.closeQuietly(HBaseFsck.this);
502         unlockHbck();
503       }
504     });
505 
506     LOG.info("Launching hbck");
507 
508     connection = (ClusterConnection)ConnectionFactory.createConnection(getConf());
509     admin = connection.getAdmin();
510     meta = connection.getTable(TableName.META_TABLE_NAME);
511     status = admin.getClusterStatus();
512   }
513 
514   /**
515    * Get deployed regions according to the region servers.
516    */
517   private void loadDeployedRegions() throws IOException, InterruptedException {
518     // From the master, get a list of all known live region servers
519     Collection<ServerName> regionServers = status.getServers();
520     errors.print("Number of live region servers: " + regionServers.size());
521     if (details) {
522       for (ServerName rsinfo: regionServers) {
523         errors.print("  " + rsinfo.getServerName());
524       }
525     }
526 
527     // From the master, get a list of all dead region servers
528     Collection<ServerName> deadRegionServers = status.getDeadServerNames();
529     errors.print("Number of dead region servers: " + deadRegionServers.size());
530     if (details) {
531       for (ServerName name: deadRegionServers) {
532         errors.print("  " + name);
533       }
534     }
535 
536     // Print the current master name and state
537     errors.print("Master: " + status.getMaster());
538 
539     // Print the list of all backup masters
540     Collection<ServerName> backupMasters = status.getBackupMasters();
541     errors.print("Number of backup masters: " + backupMasters.size());
542     if (details) {
543       for (ServerName name: backupMasters) {
544         errors.print("  " + name);
545       }
546     }
547 
548     errors.print("Average load: " + status.getAverageLoad());
549     errors.print("Number of requests: " + status.getRequestsCount());
550     errors.print("Number of regions: " + status.getRegionsCount());
551 
552     Map<String, RegionState> rits = status.getRegionsInTransition();
553     errors.print("Number of regions in transition: " + rits.size());
554     if (details) {
555       for (RegionState state: rits.values()) {
556         errors.print("  " + state.toDescriptiveString());
557       }
558     }
559 
560     // Determine what's deployed
561     processRegionServers(regionServers);
562   }
563 
564   /**
565    * Clear the current state of hbck.
566    */
567   private void clearState() {
568     // Make sure regionInfo is empty before starting
569     fixes = 0;
570     regionInfoMap.clear();
571     emptyRegionInfoQualifiers.clear();
572     disabledTables.clear();
573     errors.clear();
574     tablesInfo.clear();
575     orphanHdfsDirs.clear();
576     skippedRegions.clear();
577   }
578 
579   /**
580    * This repair method analyzes hbase data in hdfs and repairs it to satisfy
581    * the table integrity rules.  HBase doesn't need to be online for this
582    * operation to work.
583    */
584   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
585     // Initial pass to fix orphans.
586     if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
587         || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
588       LOG.info("Loading regioninfos HDFS");
589       // if nothing is happening this should always complete in two iterations.
590       int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
591       int curIter = 0;
592       do {
593         clearState(); // clears hbck state and reset fixes to 0 and.
594         // repair what's on HDFS
595         restoreHdfsIntegrity();
596         curIter++;// limit the number of iterations.
597       } while (fixes > 0 && curIter <= maxIterations);
598 
599       // Repairs should be done in the first iteration and verification in the second.
600       // If there are more than 2 passes, something funny has happened.
601       if (curIter > 2) {
602         if (curIter == maxIterations) {
603           LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
604               + "Tables integrity may not be fully repaired!");
605         } else {
606           LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
607         }
608       }
609     }
610   }
611 
612   /**
613    * This repair method requires the cluster to be online since it contacts
614    * region servers and the masters.  It makes each region's state in HDFS, in
615    * hbase:meta, and deployments consistent.
616    *
617    * @return If &gt; 0 , number of errors detected, if &lt; 0 there was an unrecoverable
618    * error.  If 0, we have a clean hbase.
619    */
620   public int onlineConsistencyRepair() throws IOException, KeeperException,
621     InterruptedException {
622     clearState();
623 
624     // get regions according to what is online on each RegionServer
625     loadDeployedRegions();
626     // check whether hbase:meta is deployed and online
627     recordMetaRegion();
628     // Check if hbase:meta is found only once and in the right place
629     if (!checkMetaRegion()) {
630       String errorMsg = "hbase:meta table is not consistent. ";
631       if (shouldFixAssignments()) {
632         errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
633       } else {
634         errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
635       }
636       errors.reportError(errorMsg + " Exiting...");
637       return -2;
638     }
639     // Not going with further consistency check for tables when hbase:meta itself is not consistent.
640     LOG.info("Loading regionsinfo from the hbase:meta table");
641     boolean success = loadMetaEntries();
642     if (!success) return -1;
643 
644     // Empty cells in hbase:meta?
645     reportEmptyMetaCells();
646 
647     // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
648     if (shouldFixEmptyMetaCells()) {
649       fixEmptyMetaCells();
650     }
651 
652     // get a list of all tables that have not changed recently.
653     if (!checkMetaOnly) {
654       reportTablesInFlux();
655     }
656 
657     // load regiondirs and regioninfos from HDFS
658     if (shouldCheckHdfs()) {
659       LOG.info("Loading region directories from HDFS");
660       loadHdfsRegionDirs();
661       LOG.info("Loading region information from HDFS");
662       loadHdfsRegionInfos();
663     }
664 
665     // Get disabled tables from ZooKeeper
666     loadDisabledTables();
667 
668     // fix the orphan tables
669     fixOrphanTables();
670 
671     LOG.info("Checking and fixing region consistency");
672     // Check and fix consistency
673     checkAndFixConsistency();
674 
675     // Check integrity (does not fix)
676     checkIntegrity();
677     return errors.getErrorList().size();
678   }
679 
680   /**
681    * Contacts the master and prints out cluster-wide information
682    * @return 0 on success, non-zero on failure
683    */
684   public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
685     // print hbase server version
686     errors.print("Version: " + status.getHBaseVersion());
687     offlineHdfsIntegrityRepair();
688 
689     boolean oldBalancer = false;
690     if (shouldDisableBalancer()) {
691       oldBalancer = admin.setBalancerRunning(false, true);
692     }
693 
694     try {
695       onlineConsistencyRepair();
696     }
697     finally {
698       // Only restore the balancer if it was true when we started repairing and
699       // we actually disabled it. Otherwise, we might clobber another run of
700       // hbck that has just restored it.
701       if (shouldDisableBalancer() && oldBalancer) {
702         admin.setBalancerRunning(oldBalancer, false);
703       }
704     }
705 
706     if (checkRegionBoundaries) {
707       checkRegionBoundaries();
708     }
709 
710     offlineReferenceFileRepair();
711 
712     checkAndFixTableLocks();
713 
714     // Check (and fix if requested) orphaned table ZNodes
715     checkAndFixOrphanedTableZNodes();
716 
717     // Remove the hbck lock
718     unlockHbck();
719 
720     // Print table summary
721     printTableSummary(tablesInfo);
722     return errors.summarize();
723   }
724 
725   public static byte[] keyOnly (byte[] b) {
726     if (b == null)
727       return b;
728     int rowlength = Bytes.toShort(b, 0);
729     byte[] result = new byte[rowlength];
730     System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
731     return result;
732   }
733 
734   @Override
735   public void close() throws IOException {
736     IOUtils.closeQuietly(admin);
737     IOUtils.closeQuietly(meta);
738     IOUtils.closeQuietly(connection);
739   }
740 
741   private static class RegionBoundariesInformation {
742     public byte [] regionName;
743     public byte [] metaFirstKey;
744     public byte [] metaLastKey;
745     public byte [] storesFirstKey;
746     public byte [] storesLastKey;
747     @Override
748     public String toString () {
749       return "regionName=" + Bytes.toStringBinary(regionName) +
750              "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
751              "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
752              "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
753              "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
754     }
755   }
756 
757   public void checkRegionBoundaries() {
758     try {
759       ByteArrayComparator comparator = new ByteArrayComparator();
760       List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), connection, false);
761       final RegionBoundariesInformation currentRegionBoundariesInformation =
762           new RegionBoundariesInformation();
763       Path hbaseRoot = FSUtils.getRootDir(getConf());
764       for (HRegionInfo regionInfo : regions) {
765         Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
766         currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
767         // For each region, get the start and stop key from the META and compare them to the
768         // same information from the Stores.
769         Path path = new Path(tableDir, regionInfo.getEncodedName());
770         FileSystem fs = path.getFileSystem(getConf());
771         FileStatus[] files = fs.listStatus(path);
772         // For all the column families in this region...
773         byte[] storeFirstKey = null;
774         byte[] storeLastKey = null;
775         for (FileStatus file : files) {
776           String fileName = file.getPath().toString();
777           fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
778           if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
779             FileStatus[] storeFiles = fs.listStatus(file.getPath());
780             // For all the stores in this column family.
781             for (FileStatus storeFile : storeFiles) {
782               HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
783                   getConf()), getConf());
784               if ((reader.getFirstKey() != null)
785                   && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
786                       reader.getFirstKey()) > 0))) {
787                 storeFirstKey = reader.getFirstKey();
788               }
789               if ((reader.getLastKey() != null)
790                   && ((storeLastKey == null) || (comparator.compare(storeLastKey,
791                       reader.getLastKey())) < 0)) {
792                 storeLastKey = reader.getLastKey();
793               }
794               reader.close();
795             }
796           }
797         }
798         currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
799         currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
800         currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
801         currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
802         if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
803           currentRegionBoundariesInformation.metaFirstKey = null;
804         if (currentRegionBoundariesInformation.metaLastKey.length == 0)
805           currentRegionBoundariesInformation.metaLastKey = null;
806 
807         // For a region to be correct, we need the META start key to be smaller or equal to the
808         // smallest start key from all the stores, and the start key from the next META entry to
809         // be bigger than the last key from all the current stores. First region start key is null;
810         // Last region end key is null; some regions can be empty and not have any store.
811 
812         boolean valid = true;
813         // Checking start key.
814         if ((currentRegionBoundariesInformation.storesFirstKey != null)
815             && (currentRegionBoundariesInformation.metaFirstKey != null)) {
816           valid = valid
817               && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
818                 currentRegionBoundariesInformation.metaFirstKey) >= 0;
819         }
820         // Checking stop key.
821         if ((currentRegionBoundariesInformation.storesLastKey != null)
822             && (currentRegionBoundariesInformation.metaLastKey != null)) {
823           valid = valid
824               && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
825                 currentRegionBoundariesInformation.metaLastKey) < 0;
826         }
827         if (!valid) {
828           errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
829             tablesInfo.get(regionInfo.getTable()));
830           LOG.warn("Region's boundaries not alligned between stores and META for:");
831           LOG.warn(currentRegionBoundariesInformation);
832         }
833       }
834     } catch (IOException e) {
835       LOG.error(e);
836     }
837   }
838 
839   /**
840    * Iterates through the list of all orphan/invalid regiondirs.
841    */
842   private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
843     for (HbckInfo hi : orphanHdfsDirs) {
844       LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
845       adoptHdfsOrphan(hi);
846     }
847   }
848 
849   /**
850    * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
851    * these orphans by creating a new region, and moving the column families,
852    * recovered edits, WALs, into the new region dir.  We determine the region
853    * startkey and endkeys by looking at all of the hfiles inside the column
854    * families to identify the min and max keys. The resulting region will
855    * likely violate table integrity but will be dealt with by merging
856    * overlapping regions.
857    */
858   @SuppressWarnings("deprecation")
859   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
860     Path p = hi.getHdfsRegionDir();
861     FileSystem fs = p.getFileSystem(getConf());
862     FileStatus[] dirs = fs.listStatus(p);
863     if (dirs == null) {
864       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
865           p + ". This dir could probably be deleted.");
866       return ;
867     }
868 
869     TableName tableName = hi.getTableName();
870     TableInfo tableInfo = tablesInfo.get(tableName);
871     Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
872     HTableDescriptor template = tableInfo.getHTD();
873 
874     // find min and max key values
875     Pair<byte[],byte[]> orphanRegionRange = null;
876     for (FileStatus cf : dirs) {
877       String cfName= cf.getPath().getName();
878       // TODO Figure out what the special dirs are
879       if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
880 
881       FileStatus[] hfiles = fs.listStatus(cf.getPath());
882       for (FileStatus hfile : hfiles) {
883         byte[] start, end;
884         HFile.Reader hf = null;
885         try {
886           CacheConfig cacheConf = new CacheConfig(getConf());
887           hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
888           hf.loadFileInfo();
889           KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
890           start = startKv.getRow();
891           KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
892           end = endKv.getRow();
893         } catch (IOException ioe) {
894           LOG.warn("Problem reading orphan file " + hfile + ", skipping");
895           continue;
896         } catch (NullPointerException ioe) {
897           LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
898           continue;
899         } finally {
900           if (hf != null) {
901             hf.close();
902           }
903         }
904 
905         // expand the range to include the range of all hfiles
906         if (orphanRegionRange == null) {
907           // first range
908           orphanRegionRange = new Pair<byte[], byte[]>(start, end);
909         } else {
910           // TODO add test
911 
912           // expand range only if the hfile is wider.
913           if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
914             orphanRegionRange.setFirst(start);
915           }
916           if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
917             orphanRegionRange.setSecond(end);
918           }
919         }
920       }
921     }
922     if (orphanRegionRange == null) {
923       LOG.warn("No data in dir " + p + ", sidelining data");
924       fixes++;
925       sidelineRegionDir(fs, hi);
926       return;
927     }
928     LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
929         Bytes.toString(orphanRegionRange.getSecond()) + ")");
930 
931     // create new region on hdfs.  move data into place.
932     HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(), 
933       Bytes.add(orphanRegionRange.getSecond(), new byte[1]));
934     LOG.info("Creating new region : " + hri);
935     HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
936     Path target = region.getRegionFileSystem().getRegionDir();
937 
938     // rename all the data to new region
939     mergeRegionDirs(target, hi);
940     fixes++;
941   }
942 
943   /**
944    * This method determines if there are table integrity errors in HDFS.  If
945    * there are errors and the appropriate "fix" options are enabled, the method
946    * will first correct orphan regions making them into legit regiondirs, and
947    * then reload to merge potentially overlapping regions.
948    *
949    * @return number of table integrity errors found
950    */
951   private int restoreHdfsIntegrity() throws IOException, InterruptedException {
952     // Determine what's on HDFS
953     LOG.info("Loading HBase regioninfo from HDFS...");
954     loadHdfsRegionDirs(); // populating regioninfo table.
955 
956     int errs = errors.getErrorList().size();
957     // First time just get suggestions.
958     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
959     checkHdfsIntegrity(false, false);
960 
961     if (errors.getErrorList().size() == errs) {
962       LOG.info("No integrity errors.  We are done with this phase. Glorious.");
963       return 0;
964     }
965 
966     if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
967       adoptHdfsOrphans(orphanHdfsDirs);
968       // TODO optimize by incrementally adding instead of reloading.
969     }
970 
971     // Make sure there are no holes now.
972     if (shouldFixHdfsHoles()) {
973       clearState(); // this also resets # fixes.
974       loadHdfsRegionDirs();
975       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
976       tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
977     }
978 
979     // Now we fix overlaps
980     if (shouldFixHdfsOverlaps()) {
981       // second pass we fix overlaps.
982       clearState(); // this also resets # fixes.
983       loadHdfsRegionDirs();
984       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
985       tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
986     }
987 
988     return errors.getErrorList().size();
989   }
990 
991   /**
992    * Scan all the store file names to find any lingering reference files,
993    * which refer to some none-exiting files. If "fix" option is enabled,
994    * any lingering reference file will be sidelined if found.
995    * <p>
996    * Lingering reference file prevents a region from opening. It has to
997    * be fixed before a cluster can start properly.
998    */
999   private void offlineReferenceFileRepair() throws IOException {
1000     Configuration conf = getConf();
1001     Path hbaseRoot = FSUtils.getRootDir(conf);
1002     FileSystem fs = hbaseRoot.getFileSystem(conf);
1003     LOG.info("Computing mapping of all store files");
1004     Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot, errors);
1005     errors.print("");
1006     LOG.info("Validating mapping using HDFS state");
1007     for (Path path: allFiles.values()) {
1008       boolean isReference = false;
1009       try {
1010         isReference = StoreFileInfo.isReference(path);
1011       } catch (Throwable t) {
1012         // Ignore. Some files may not be store files at all.
1013         // For example, files under .oldlogs folder in hbase:meta
1014         // Warning message is already logged by
1015         // StoreFile#isReference.
1016       }
1017       if (!isReference) continue;
1018 
1019       Path referredToFile = StoreFileInfo.getReferredToFile(path);
1020       if (fs.exists(referredToFile)) continue;  // good, expected
1021 
1022       // Found a lingering reference file
1023       errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
1024         "Found lingering reference file " + path);
1025       if (!shouldFixReferenceFiles()) continue;
1026 
1027       // Now, trying to fix it since requested
1028       boolean success = false;
1029       String pathStr = path.toString();
1030 
1031       // A reference file path should be like
1032       // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
1033       // Up 5 directories to get the root folder.
1034       // So the file will be sidelined to a similar folder structure.
1035       int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
1036       for (int i = 0; index > 0 && i < 5; i++) {
1037         index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
1038       }
1039       if (index > 0) {
1040         Path rootDir = getSidelineDir();
1041         Path dst = new Path(rootDir, pathStr.substring(index + 1));
1042         fs.mkdirs(dst.getParent());
1043         LOG.info("Trying to sildeline reference file "
1044           + path + " to " + dst);
1045         setShouldRerun();
1046 
1047         success = fs.rename(path, dst);
1048         debugLsr(dst);
1049 
1050       }
1051       if (!success) {
1052         LOG.error("Failed to sideline reference file " + path);
1053       }
1054     }
1055   }
1056 
1057   /**
1058    * TODO -- need to add tests for this.
1059    */
1060   private void reportEmptyMetaCells() {
1061     errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
1062       emptyRegionInfoQualifiers.size());
1063     if (details) {
1064       for (Result r: emptyRegionInfoQualifiers) {
1065         errors.print("  " + r);
1066       }
1067     }
1068   }
1069 
1070   /**
1071    * TODO -- need to add tests for this.
1072    */
1073   private void reportTablesInFlux() {
1074     AtomicInteger numSkipped = new AtomicInteger(0);
1075     HTableDescriptor[] allTables = getTables(numSkipped);
1076     errors.print("Number of Tables: " + allTables.length);
1077     if (details) {
1078       if (numSkipped.get() > 0) {
1079         errors.detail("Number of Tables in flux: " + numSkipped.get());
1080       }
1081       for (HTableDescriptor td : allTables) {
1082         errors.detail("  Table: " + td.getTableName() + "\t" +
1083                            (td.isReadOnly() ? "ro" : "rw") + "\t" +
1084                             (td.isMetaRegion() ? "META" : "    ") + "\t" +
1085                            " families: " + td.getFamilies().size());
1086       }
1087     }
1088   }
1089 
1090   public ErrorReporter getErrors() {
1091     return errors;
1092   }
1093 
1094   /**
1095    * Read the .regioninfo file from the file system.  If there is no
1096    * .regioninfo, add it to the orphan hdfs region list.
1097    */
1098   private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
1099     Path regionDir = hbi.getHdfsRegionDir();
1100     if (regionDir == null) {
1101       if (hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1102         // Log warning only for default/ primary replica with no region dir
1103         LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
1104       }
1105       return;
1106     }
1107 
1108     if (hbi.hdfsEntry.hri != null) {
1109       // already loaded data
1110       return;
1111     }
1112 
1113     FileSystem fs = FileSystem.get(getConf());
1114     HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
1115     LOG.debug("HRegionInfo read: " + hri.toString());
1116     hbi.hdfsEntry.hri = hri;
1117   }
1118 
1119   /**
1120    * Exception thrown when a integrity repair operation fails in an
1121    * unresolvable way.
1122    */
1123   public static class RegionRepairException extends IOException {
1124     private static final long serialVersionUID = 1L;
1125     final IOException ioe;
1126     public RegionRepairException(String s, IOException ioe) {
1127       super(s);
1128       this.ioe = ioe;
1129     }
1130   }
1131 
1132   /**
1133    * Populate hbi's from regionInfos loaded from file system.
1134    */
1135   private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
1136       throws IOException, InterruptedException {
1137     tablesInfo.clear(); // regenerating the data
1138     // generate region split structure
1139     Collection<HbckInfo> hbckInfos = regionInfoMap.values();
1140 
1141     // Parallelized read of .regioninfo files.
1142     List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
1143     List<Future<Void>> hbiFutures;
1144 
1145     for (HbckInfo hbi : hbckInfos) {
1146       WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1147       hbis.add(work);
1148     }
1149 
1150     // Submit and wait for completion
1151     hbiFutures = executor.invokeAll(hbis);
1152 
1153     for(int i=0; i<hbiFutures.size(); i++) {
1154       WorkItemHdfsRegionInfo work = hbis.get(i);
1155       Future<Void> f = hbiFutures.get(i);
1156       try {
1157         f.get();
1158       } catch(ExecutionException e) {
1159         LOG.warn("Failed to read .regioninfo file for region " +
1160               work.hbi.getRegionNameAsString(), e.getCause());
1161       }
1162     }
1163 
1164     Path hbaseRoot = FSUtils.getRootDir(getConf());
1165     FileSystem fs = hbaseRoot.getFileSystem(getConf());
1166     // serialized table info gathering.
1167     for (HbckInfo hbi: hbckInfos) {
1168 
1169       if (hbi.getHdfsHRI() == null) {
1170         // was an orphan
1171         continue;
1172       }
1173 
1174 
1175       // get table name from hdfs, populate various HBaseFsck tables.
1176       TableName tableName = hbi.getTableName();
1177       if (tableName == null) {
1178         // There was an entry in hbase:meta not in the HDFS?
1179         LOG.warn("tableName was null for: " + hbi);
1180         continue;
1181       }
1182 
1183       TableInfo modTInfo = tablesInfo.get(tableName);
1184       if (modTInfo == null) {
1185         // only executed once per table.
1186         modTInfo = new TableInfo(tableName);
1187         tablesInfo.put(tableName, modTInfo);
1188         try {
1189           HTableDescriptor htd =
1190               FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1191           modTInfo.htds.add(htd);
1192         } catch (IOException ioe) {
1193           if (!orphanTableDirs.containsKey(tableName)) {
1194             LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1195             //should only report once for each table
1196             errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1197                 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1198             Set<String> columns = new HashSet<String>();
1199             orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1200           }
1201         }
1202       }
1203       if (!hbi.isSkipChecks()) {
1204         modTInfo.addRegionInfo(hbi);
1205       }
1206     }
1207 
1208     loadTableInfosForTablesWithNoRegion();
1209     errors.print("");
1210 
1211     return tablesInfo;
1212   }
1213 
1214   /**
1215    * To get the column family list according to the column family dirs
1216    * @param columns
1217    * @param hbi
1218    * @return a set of column families
1219    * @throws IOException
1220    */
1221   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
1222     Path regionDir = hbi.getHdfsRegionDir();
1223     FileSystem fs = regionDir.getFileSystem(getConf());
1224     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1225     for (FileStatus subdir : subDirs) {
1226       String columnfamily = subdir.getPath().getName();
1227       columns.add(columnfamily);
1228     }
1229     return columns;
1230   }
1231 
1232   /**
1233    * To fabricate a .tableinfo file with following contents<br>
1234    * 1. the correct tablename <br>
1235    * 2. the correct colfamily list<br>
1236    * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1237    * @throws IOException
1238    */
1239   private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1240       Set<String> columns) throws IOException {
1241     if (columns ==null || columns.isEmpty()) return false;
1242     HTableDescriptor htd = new HTableDescriptor(tableName);
1243     for (String columnfamimly : columns) {
1244       htd.addFamily(new HColumnDescriptor(columnfamimly));
1245     }
1246     fstd.createTableDescriptor(htd, true);
1247     return true;
1248   }
1249 
1250   /**
1251    * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1252    * @throws IOException
1253    */
1254   public void fixEmptyMetaCells() throws IOException {
1255     if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1256       LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1257       for (Result region : emptyRegionInfoQualifiers) {
1258         deleteMetaRegion(region.getRow());
1259         errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1260       }
1261       emptyRegionInfoQualifiers.clear();
1262     }
1263   }
1264 
1265   /**
1266    * To fix orphan table by creating a .tableinfo file under tableDir <br>
1267    * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1268    * 2. else create a default .tableinfo file with following items<br>
1269    * &nbsp;2.1 the correct tablename <br>
1270    * &nbsp;2.2 the correct colfamily list<br>
1271    * &nbsp;2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1272    * @throws IOException
1273    */
1274   public void fixOrphanTables() throws IOException {
1275     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1276 
1277       List<TableName> tmpList = new ArrayList<TableName>();
1278       tmpList.addAll(orphanTableDirs.keySet());
1279       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1280       Iterator<Entry<TableName, Set<String>>> iter =
1281           orphanTableDirs.entrySet().iterator();
1282       int j = 0;
1283       int numFailedCase = 0;
1284       FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1285       while (iter.hasNext()) {
1286         Entry<TableName, Set<String>> entry =
1287             iter.next();
1288         TableName tableName = entry.getKey();
1289         LOG.info("Trying to fix orphan table error: " + tableName);
1290         if (j < htds.length) {
1291           if (tableName.equals(htds[j].getTableName())) {
1292             HTableDescriptor htd = htds[j];
1293             LOG.info("fixing orphan table: " + tableName + " from cache");
1294             fstd.createTableDescriptor(htd, true);
1295             j++;
1296             iter.remove();
1297           }
1298         } else {
1299           if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1300             LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1301             LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1302             iter.remove();
1303           } else {
1304             LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1305             numFailedCase++;
1306           }
1307         }
1308         fixes++;
1309       }
1310 
1311       if (orphanTableDirs.isEmpty()) {
1312         // all orphanTableDirs are luckily recovered
1313         // re-run doFsck after recovering the .tableinfo file
1314         setShouldRerun();
1315         LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1316       } else if (numFailedCase > 0) {
1317         LOG.error("Failed to fix " + numFailedCase
1318             + " OrphanTables with default .tableinfo files");
1319       }
1320 
1321     }
1322     //cleanup the list
1323     orphanTableDirs.clear();
1324 
1325   }
1326 
1327   /**
1328    * This borrows code from MasterFileSystem.bootstrap()
1329    *
1330    * @return an open hbase:meta HRegion
1331    */
1332   private HRegion createNewMeta() throws IOException {
1333       Path rootdir = FSUtils.getRootDir(getConf());
1334     Configuration c = getConf();
1335     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1336     HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
1337     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false);
1338     HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor);
1339     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
1340     return meta;
1341   }
1342 
1343   /**
1344    * Generate set of puts to add to new meta.  This expects the tables to be
1345    * clean with no overlaps or holes.  If there are any problems it returns null.
1346    *
1347    * @return An array list of puts to do in bulk, null if tables have problems
1348    */
1349   private ArrayList<Put> generatePuts(
1350       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1351     ArrayList<Put> puts = new ArrayList<Put>();
1352     boolean hasProblems = false;
1353     for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1354       TableName name = e.getKey();
1355 
1356       // skip "hbase:meta"
1357       if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1358         continue;
1359       }
1360 
1361       TableInfo ti = e.getValue();
1362       for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1363           .entrySet()) {
1364         Collection<HbckInfo> his = spl.getValue();
1365         int sz = his.size();
1366         if (sz != 1) {
1367           // problem
1368           LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1369               + " had " +  sz + " regions instead of exactly 1." );
1370           hasProblems = true;
1371           continue;
1372         }
1373 
1374         // add the row directly to meta.
1375         HbckInfo hi = his.iterator().next();
1376         HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
1377         Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1378         puts.add(p);
1379       }
1380     }
1381     return hasProblems ? null : puts;
1382   }
1383 
1384   /**
1385    * Suggest fixes for each table
1386    */
1387   private void suggestFixes(
1388       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1389     logParallelMerge();
1390     for (TableInfo tInfo : tablesInfo.values()) {
1391       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1392       tInfo.checkRegionChain(handler);
1393     }
1394   }
1395 
1396   /**
1397    * Rebuilds meta from information in hdfs/fs.  Depends on configuration
1398    * settings passed into hbck constructor to point to a particular fs/dir.
1399    *
1400    * @param fix flag that determines if method should attempt to fix holes
1401    * @return true if successful, false if attempt failed.
1402    */
1403   public boolean rebuildMeta(boolean fix) throws IOException,
1404       InterruptedException {
1405 
1406     // TODO check to make sure hbase is offline. (or at least the table
1407     // currently being worked on is off line)
1408 
1409     // Determine what's on HDFS
1410     LOG.info("Loading HBase regioninfo from HDFS...");
1411     loadHdfsRegionDirs(); // populating regioninfo table.
1412 
1413     int errs = errors.getErrorList().size();
1414     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1415     checkHdfsIntegrity(false, false);
1416 
1417     // make sure ok.
1418     if (errors.getErrorList().size() != errs) {
1419       // While in error state, iterate until no more fixes possible
1420       while(true) {
1421         fixes = 0;
1422         suggestFixes(tablesInfo);
1423         errors.clear();
1424         loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1425         checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1426 
1427         int errCount = errors.getErrorList().size();
1428 
1429         if (fixes == 0) {
1430           if (errCount > 0) {
1431             return false; // failed to fix problems.
1432           } else {
1433             break; // no fixes and no problems? drop out and fix stuff!
1434           }
1435         }
1436       }
1437     }
1438 
1439     // we can rebuild, move old meta out of the way and start
1440     LOG.info("HDFS regioninfo's seems good.  Sidelining old hbase:meta");
1441     Path backupDir = sidelineOldMeta();
1442 
1443     LOG.info("Creating new hbase:meta");
1444     HRegion meta = createNewMeta();
1445 
1446     // populate meta
1447     List<Put> puts = generatePuts(tablesInfo);
1448     if (puts == null) {
1449       LOG.fatal("Problem encountered when creating new hbase:meta entries.  " +
1450         "You may need to restore the previously sidelined hbase:meta");
1451       return false;
1452     }
1453     meta.batchMutate(puts.toArray(new Put[puts.size()]));
1454     HRegion.closeHRegion(meta);
1455     LOG.info("Success! hbase:meta table rebuilt.");
1456     LOG.info("Old hbase:meta is moved into " + backupDir);
1457     return true;
1458   }
1459 
1460   /**
1461    * Log an appropriate message about whether or not overlapping merges are computed in parallel.
1462    */
1463   private void logParallelMerge() {
1464     if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
1465       LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
1466           " false to run serially.");
1467     } else {
1468       LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
1469           " true to run in parallel.");
1470     }
1471   }
1472 
1473   private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1474       boolean fixOverlaps) throws IOException {
1475     LOG.info("Checking HBase region split map from HDFS data...");
1476     logParallelMerge();
1477     for (TableInfo tInfo : tablesInfo.values()) {
1478       TableIntegrityErrorHandler handler;
1479       if (fixHoles || fixOverlaps) {
1480         handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1481           fixHoles, fixOverlaps);
1482       } else {
1483         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1484       }
1485       if (!tInfo.checkRegionChain(handler)) {
1486         // should dump info as well.
1487         errors.report("Found inconsistency in table " + tInfo.getName());
1488       }
1489     }
1490     return tablesInfo;
1491   }
1492 
1493   private Path getSidelineDir() throws IOException {
1494     if (sidelineDir == null) {
1495       Path hbaseDir = FSUtils.getRootDir(getConf());
1496       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1497       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1498           + startMillis);
1499     }
1500     return sidelineDir;
1501   }
1502 
1503   /**
1504    * Sideline a region dir (instead of deleting it)
1505    */
1506   Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1507     return sidelineRegionDir(fs, null, hi);
1508   }
1509 
1510   /**
1511    * Sideline a region dir (instead of deleting it)
1512    *
1513    * @param parentDir if specified, the region will be sidelined to
1514    * folder like .../parentDir/<table name>/<region name>. The purpose
1515    * is to group together similar regions sidelined, for example, those
1516    * regions should be bulk loaded back later on. If null, it is ignored.
1517    */
1518   Path sidelineRegionDir(FileSystem fs,
1519       String parentDir, HbckInfo hi) throws IOException {
1520     TableName tableName = hi.getTableName();
1521     Path regionDir = hi.getHdfsRegionDir();
1522 
1523     if (!fs.exists(regionDir)) {
1524       LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1525       return null;
1526     }
1527 
1528     Path rootDir = getSidelineDir();
1529     if (parentDir != null) {
1530       rootDir = new Path(rootDir, parentDir);
1531     }
1532     Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1533     Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1534     fs.mkdirs(sidelineRegionDir);
1535     boolean success = false;
1536     FileStatus[] cfs =  fs.listStatus(regionDir);
1537     if (cfs == null) {
1538       LOG.info("Region dir is empty: " + regionDir);
1539     } else {
1540       for (FileStatus cf : cfs) {
1541         Path src = cf.getPath();
1542         Path dst =  new Path(sidelineRegionDir, src.getName());
1543         if (fs.isFile(src)) {
1544           // simple file
1545           success = fs.rename(src, dst);
1546           if (!success) {
1547             String msg = "Unable to rename file " + src +  " to " + dst;
1548             LOG.error(msg);
1549             throw new IOException(msg);
1550           }
1551           continue;
1552         }
1553 
1554         // is a directory.
1555         fs.mkdirs(dst);
1556 
1557         LOG.info("Sidelining files from " + src + " into containing region " + dst);
1558         // FileSystem.rename is inconsistent with directories -- if the
1559         // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1560         // it moves the src into the dst dir resulting in (foo/a/b).  If
1561         // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1562         FileStatus[] hfiles = fs.listStatus(src);
1563         if (hfiles != null && hfiles.length > 0) {
1564           for (FileStatus hfile : hfiles) {
1565             success = fs.rename(hfile.getPath(), dst);
1566             if (!success) {
1567               String msg = "Unable to rename file " + src +  " to " + dst;
1568               LOG.error(msg);
1569               throw new IOException(msg);
1570             }
1571           }
1572         }
1573         LOG.debug("Sideline directory contents:");
1574         debugLsr(sidelineRegionDir);
1575       }
1576     }
1577 
1578     LOG.info("Removing old region dir: " + regionDir);
1579     success = fs.delete(regionDir, true);
1580     if (!success) {
1581       String msg = "Unable to delete dir " + regionDir;
1582       LOG.error(msg);
1583       throw new IOException(msg);
1584     }
1585     return sidelineRegionDir;
1586   }
1587 
1588   /**
1589    * Side line an entire table.
1590    */
1591   void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1592       Path backupHbaseDir) throws IOException {
1593     Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1594     if (fs.exists(tableDir)) {
1595       Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1596       fs.mkdirs(backupTableDir.getParent());
1597       boolean success = fs.rename(tableDir, backupTableDir);
1598       if (!success) {
1599         throw new IOException("Failed to move  " + tableName + " from "
1600             +  tableDir + " to " + backupTableDir);
1601       }
1602     } else {
1603       LOG.info("No previous " + tableName +  " exists.  Continuing.");
1604     }
1605   }
1606 
1607   /**
1608    * @return Path to backup of original directory
1609    */
1610   Path sidelineOldMeta() throws IOException {
1611     // put current hbase:meta aside.
1612     Path hbaseDir = FSUtils.getRootDir(getConf());
1613     FileSystem fs = hbaseDir.getFileSystem(getConf());
1614     Path backupDir = getSidelineDir();
1615     fs.mkdirs(backupDir);
1616 
1617     try {
1618       sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1619     } catch (IOException e) {
1620         LOG.fatal("... failed to sideline meta. Currently in inconsistent state.  To restore "
1621             + "try to rename hbase:meta in " + backupDir.getName() + " to "
1622             + hbaseDir.getName() + ".", e);
1623       throw e; // throw original exception
1624     }
1625     return backupDir;
1626   }
1627 
1628   /**
1629    * Load the list of disabled tables in ZK into local set.
1630    * @throws ZooKeeperConnectionException
1631    * @throws IOException
1632    */
1633   private void loadDisabledTables()
1634   throws ZooKeeperConnectionException, IOException {
1635     HConnectionManager.execute(new HConnectable<Void>(getConf()) {
1636       @Override
1637       public Void connect(HConnection connection) throws IOException {
1638         ZooKeeperWatcher zkw = createZooKeeperWatcher();
1639         try {
1640           for (TableName tableName :
1641               ZKTableStateClientSideReader.getDisabledOrDisablingTables(zkw)) {
1642             disabledTables.add(tableName);
1643           }
1644         } catch (KeeperException ke) {
1645           throw new IOException(ke);
1646         } catch (InterruptedException e) {
1647           throw new InterruptedIOException();
1648         } finally {
1649           zkw.close();
1650         }
1651         return null;
1652       }
1653     });
1654   }
1655 
1656   /**
1657    * Check if the specified region's table is disabled.
1658    */
1659   private boolean isTableDisabled(HRegionInfo regionInfo) {
1660     return disabledTables.contains(regionInfo.getTable());
1661   }
1662 
1663   /**
1664    * Scan HDFS for all regions, recording their information into
1665    * regionInfoMap
1666    */
1667   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1668     Path rootDir = FSUtils.getRootDir(getConf());
1669     FileSystem fs = rootDir.getFileSystem(getConf());
1670 
1671     // list all tables from HDFS
1672     List<FileStatus> tableDirs = Lists.newArrayList();
1673 
1674     boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1675 
1676     List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1677     for (Path path : paths) {
1678       TableName tableName = FSUtils.getTableName(path);
1679        if ((!checkMetaOnly &&
1680            isTableIncluded(tableName)) ||
1681            tableName.equals(TableName.META_TABLE_NAME)) {
1682          tableDirs.add(fs.getFileStatus(path));
1683        }
1684     }
1685 
1686     // verify that version file exists
1687     if (!foundVersionFile) {
1688       errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1689           "Version file does not exist in root dir " + rootDir);
1690       if (shouldFixVersionFile()) {
1691         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1692             + " file.");
1693         setShouldRerun();
1694         FSUtils.setVersion(fs, rootDir, getConf().getInt(
1695             HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1696             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1697             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1698       }
1699     }
1700 
1701     // level 1:  <HBASE_DIR>/*
1702     List<WorkItemHdfsDir> dirs = new ArrayList<WorkItemHdfsDir>(tableDirs.size());
1703     List<Future<Void>> dirsFutures;
1704 
1705     for (FileStatus tableDir : tableDirs) {
1706       LOG.debug("Loading region dirs from " +tableDir.getPath());
1707       dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir));
1708     }
1709 
1710     // Invoke and wait for Callables to complete
1711     dirsFutures = executor.invokeAll(dirs);
1712 
1713     for(Future<Void> f: dirsFutures) {
1714       try {
1715         f.get();
1716       } catch(ExecutionException e) {
1717         LOG.warn("Could not load region dir " , e.getCause());
1718       }
1719     }
1720     errors.print("");
1721   }
1722 
1723   /**
1724    * Record the location of the hbase:meta region as found in ZooKeeper.
1725    */
1726   private boolean recordMetaRegion() throws IOException {
1727     RegionLocations rl = ((ClusterConnection)connection).locateRegion(TableName.META_TABLE_NAME,
1728         HConstants.EMPTY_START_ROW, false, false);
1729     if (rl == null) {
1730       errors.reportError(ERROR_CODE.NULL_META_REGION,
1731           "META region or some of its attributes are null.");
1732       return false;
1733     }
1734     for (HRegionLocation metaLocation : rl.getRegionLocations()) {
1735       // Check if Meta region is valid and existing
1736       if (metaLocation == null || metaLocation.getRegionInfo() == null ||
1737           metaLocation.getHostname() == null) {
1738         errors.reportError(ERROR_CODE.NULL_META_REGION,
1739             "META region or some of its attributes are null.");
1740         return false;
1741       }
1742       ServerName sn = metaLocation.getServerName();
1743       MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, EnvironmentEdgeManager.currentTime());
1744       HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1745       if (hbckInfo == null) {
1746         regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1747       } else {
1748         hbckInfo.metaEntry = m;
1749       }
1750     }
1751     return true;
1752   }
1753 
1754   private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1755     return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1756       @Override
1757       public void abort(String why, Throwable e) {
1758         LOG.error(why, e);
1759         System.exit(1);
1760       }
1761 
1762       @Override
1763       public boolean isAborted() {
1764         return false;
1765       }
1766 
1767     });
1768   }
1769 
1770   private ServerName getMetaRegionServerName(int replicaId)
1771   throws IOException, KeeperException {
1772     ZooKeeperWatcher zkw = createZooKeeperWatcher();
1773     ServerName sn = null;
1774     try {
1775       sn = new MetaTableLocator().getMetaRegionLocation(zkw, replicaId);
1776     } finally {
1777       zkw.close();
1778     }
1779     return sn;
1780   }
1781 
1782   /**
1783    * Contacts each regionserver and fetches metadata about regions.
1784    * @param regionServerList - the list of region servers to connect to
1785    * @throws IOException if a remote or network exception occurs
1786    */
1787   void processRegionServers(Collection<ServerName> regionServerList)
1788     throws IOException, InterruptedException {
1789 
1790     List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1791     List<Future<Void>> workFutures;
1792 
1793     // loop to contact each region server in parallel
1794     for (ServerName rsinfo: regionServerList) {
1795       workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1796     }
1797 
1798     workFutures = executor.invokeAll(workItems);
1799 
1800     for(int i=0; i<workFutures.size(); i++) {
1801       WorkItemRegion item = workItems.get(i);
1802       Future<Void> f = workFutures.get(i);
1803       try {
1804         f.get();
1805       } catch(ExecutionException e) {
1806         LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1807             e.getCause());
1808       }
1809     }
1810   }
1811 
1812   /**
1813    * Check consistency of all regions that have been found in previous phases.
1814    */
1815   private void checkAndFixConsistency()
1816   throws IOException, KeeperException, InterruptedException {
1817 	  // Divide the checks in two phases. One for default/primary replicas and another
1818 	  // for the non-primary ones. Keeps code cleaner this way.
1819     List<CheckRegionConsistencyWorkItem> workItems =
1820         new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1821     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1822       if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1823         workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1824       }
1825     }
1826     checkRegionConsistencyConcurrently(workItems);
1827 
1828     boolean prevHdfsCheck = shouldCheckHdfs();
1829     setCheckHdfs(false); //replicas don't have any hdfs data
1830     // Run a pass over the replicas and fix any assignment issues that exist on the currently
1831     // deployed/undeployed replicas.
1832     List<CheckRegionConsistencyWorkItem> replicaWorkItems =
1833         new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size());
1834     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1835       if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1836         replicaWorkItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));
1837       }
1838     }
1839     checkRegionConsistencyConcurrently(replicaWorkItems);
1840     setCheckHdfs(prevHdfsCheck);
1841 
1842     // If some regions is skipped during checkRegionConsistencyConcurrently() phase, we might
1843     // not get accurate state of the hbase if continuing. The config here allows users to tune
1844     // the tolerance of number of skipped region.
1845     // TODO: evaluate the consequence to continue the hbck operation without config.
1846     int terminateThreshold =  getConf().getInt("hbase.hbck.skipped.regions.limit", 0);
1847     int numOfSkippedRegions = skippedRegions.size();
1848     if (numOfSkippedRegions > 0 && numOfSkippedRegions > terminateThreshold) {
1849       throw new IOException(numOfSkippedRegions
1850         + " region(s) could not be checked or repaired.  See logs for detail.");
1851     }
1852   }
1853 
1854   /**
1855    * Check consistency of all regions using mulitple threads concurrently.
1856    */
1857   private void checkRegionConsistencyConcurrently(
1858     final List<CheckRegionConsistencyWorkItem> workItems)
1859     throws IOException, KeeperException, InterruptedException {
1860     if (workItems.isEmpty()) {
1861       return;  // nothing to check
1862     }
1863 
1864     List<Future<Void>> workFutures = executor.invokeAll(workItems);
1865     for(Future<Void> f: workFutures) {
1866       try {
1867         f.get();
1868       } catch(ExecutionException e1) {
1869         LOG.warn("Could not check region consistency " , e1.getCause());
1870         if (e1.getCause() instanceof IOException) {
1871           throw (IOException)e1.getCause();
1872         } else if (e1.getCause() instanceof KeeperException) {
1873           throw (KeeperException)e1.getCause();
1874         } else if (e1.getCause() instanceof InterruptedException) {
1875           throw (InterruptedException)e1.getCause();
1876         } else {
1877           throw new IOException(e1.getCause());
1878         }
1879       }
1880     }
1881   }
1882 
1883   class CheckRegionConsistencyWorkItem implements Callable<Void> {
1884     private final String key;
1885     private final HbckInfo hbi;
1886 
1887     CheckRegionConsistencyWorkItem(String key, HbckInfo hbi) {
1888       this.key = key;
1889       this.hbi = hbi;
1890     }
1891 
1892     @Override
1893     public synchronized Void call() throws Exception {
1894       try {
1895         checkRegionConsistency(key, hbi);
1896       } catch (Exception e) {
1897         // If the region is non-META region, skip this region and send warning/error message; if
1898         // the region is META region, we should not continue.
1899         LOG.warn("Unable to complete check or repair the region '" + hbi.getRegionNameAsString()
1900           + "'.", e);
1901         if (hbi.getHdfsHRI().isMetaRegion()) {
1902           throw e;
1903         }
1904         LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'");
1905         addSkippedRegion(hbi);
1906       }
1907       return null;
1908     }
1909   }
1910 
1911   private void addSkippedRegion(final HbckInfo hbi) {
1912     Set<String> skippedRegionNames = skippedRegions.get(hbi.getTableName());
1913     if (skippedRegionNames == null) {
1914       skippedRegionNames = new HashSet<String>();
1915     }
1916     skippedRegionNames.add(hbi.getRegionNameAsString());
1917     skippedRegions.put(hbi.getTableName(), skippedRegionNames);
1918   }
1919 
1920   private void preCheckPermission() throws IOException, AccessDeniedException {
1921     if (shouldIgnorePreCheckPermission()) {
1922       return;
1923     }
1924 
1925     Path hbaseDir = FSUtils.getRootDir(getConf());
1926     FileSystem fs = hbaseDir.getFileSystem(getConf());
1927     UserProvider userProvider = UserProvider.instantiate(getConf());
1928     UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1929     FileStatus[] files = fs.listStatus(hbaseDir);
1930     for (FileStatus file : files) {
1931       try {
1932         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1933       } catch (AccessDeniedException ace) {
1934         LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1935         errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1936           + " does not have write perms to " + file.getPath()
1937           + ". Please rerun hbck as hdfs user " + file.getOwner());
1938         throw ace;
1939       }
1940     }
1941   }
1942 
1943   /**
1944    * Deletes region from meta table
1945    */
1946   private void deleteMetaRegion(HbckInfo hi) throws IOException {
1947     deleteMetaRegion(hi.metaEntry.getRegionName());
1948   }
1949 
1950   /**
1951    * Deletes region from meta table
1952    */
1953   private void deleteMetaRegion(byte[] metaKey) throws IOException {
1954     Delete d = new Delete(metaKey);
1955     meta.delete(d);
1956     LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1957   }
1958 
1959   /**
1960    * Reset the split parent region info in meta table
1961    */
1962   private void resetSplitParent(HbckInfo hi) throws IOException {
1963     RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
1964     Delete d = new Delete(hi.metaEntry.getRegionName());
1965     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1966     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1967     mutations.add(d);
1968 
1969     HRegionInfo hri = new HRegionInfo(hi.metaEntry);
1970     hri.setOffline(false);
1971     hri.setSplit(false);
1972     Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1973     mutations.add(p);
1974 
1975     meta.mutateRow(mutations);
1976     LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
1977   }
1978 
1979   /**
1980    * This backwards-compatibility wrapper for permanently offlining a region
1981    * that should not be alive.  If the region server does not support the
1982    * "offline" method, it will use the closest unassign method instead.  This
1983    * will basically work until one attempts to disable or delete the affected
1984    * table.  The problem has to do with in-memory only master state, so
1985    * restarting the HMaster or failing over to another should fix this.
1986    */
1987   private void offline(byte[] regionName) throws IOException {
1988     String regionString = Bytes.toStringBinary(regionName);
1989     if (!rsSupportsOffline) {
1990       LOG.warn("Using unassign region " + regionString
1991           + " instead of using offline method, you should"
1992           + " restart HMaster after these repairs");
1993       admin.unassign(regionName, true);
1994       return;
1995     }
1996 
1997     // first time we assume the rs's supports #offline.
1998     try {
1999       LOG.info("Offlining region " + regionString);
2000       admin.offline(regionName);
2001     } catch (IOException ioe) {
2002       String notFoundMsg = "java.lang.NoSuchMethodException: " +
2003         "org.apache.hadoop.hbase.master.HMaster.offline([B)";
2004       if (ioe.getMessage().contains(notFoundMsg)) {
2005         LOG.warn("Using unassign region " + regionString
2006             + " instead of using offline method, you should"
2007             + " restart HMaster after these repairs");
2008         rsSupportsOffline = false; // in the future just use unassign
2009         admin.unassign(regionName, true);
2010         return;
2011       }
2012       throw ioe;
2013     }
2014   }
2015 
2016   private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
2017     undeployRegionsForHbi(hi);
2018     // undeploy replicas of the region (but only if the method is invoked for the primary)
2019     if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2020       return;
2021     }
2022     int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2023     for (int i = 1; i < numReplicas; i++) {
2024       if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
2025       HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
2026           hi.getPrimaryHRIForDeployedReplica(), i);
2027       HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2028       if (h != null) {
2029         undeployRegionsForHbi(h);
2030         //set skip checks; we undeployed it, and we don't want to evaluate this anymore
2031         //in consistency checks
2032         h.setSkipChecks(true);
2033       }
2034     }
2035   }
2036 
2037   private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
2038     for (OnlineEntry rse : hi.deployedEntries) {
2039       LOG.debug("Undeploy region "  + rse.hri + " from " + rse.hsa);
2040       try {
2041         HBaseFsckRepair.closeRegionSilentlyAndWait(connection, rse.hsa, rse.hri);
2042         offline(rse.hri.getRegionName());
2043       } catch (IOException ioe) {
2044         LOG.warn("Got exception when attempting to offline region "
2045             + Bytes.toString(rse.hri.getRegionName()), ioe);
2046       }
2047     }
2048   }
2049 
2050   /**
2051    * Attempts to undeploy a region from a region server based in information in
2052    * META.  Any operations that modify the file system should make sure that
2053    * its corresponding region is not deployed to prevent data races.
2054    *
2055    * A separate call is required to update the master in-memory region state
2056    * kept in the AssignementManager.  Because disable uses this state instead of
2057    * that found in META, we can't seem to cleanly disable/delete tables that
2058    * have been hbck fixed.  When used on a version of HBase that does not have
2059    * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master
2060    * restart or failover may be required.
2061    */
2062   private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
2063     if (hi.metaEntry == null && hi.hdfsEntry == null) {
2064       undeployRegions(hi);
2065       return;
2066     }
2067 
2068     // get assignment info and hregioninfo from meta.
2069     Get get = new Get(hi.getRegionName());
2070     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
2071     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
2072     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
2073     // also get the locations of the replicas to close if the primary region is being closed
2074     if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2075       int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
2076       for (int i = 0; i < numReplicas; i++) {
2077         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
2078         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
2079       }
2080     }
2081     Result r = meta.get(get);
2082     RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
2083     if (rl == null) {
2084       LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
2085           " since meta does not have handle to reach it");
2086       return;
2087     }
2088     for (HRegionLocation h : rl.getRegionLocations()) {
2089       ServerName serverName = h.getServerName();
2090       if (serverName == null) {
2091         errors.reportError("Unable to close region "
2092             + hi.getRegionNameAsString() +  " because meta does not "
2093             + "have handle to reach it.");
2094         continue;
2095       }
2096       HRegionInfo hri = h.getRegionInfo();
2097       if (hri == null) {
2098         LOG.warn("Unable to close region " + hi.getRegionNameAsString()
2099             + " because hbase:meta had invalid or missing "
2100             + HConstants.CATALOG_FAMILY_STR + ":"
2101             + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
2102             + " qualifier value.");
2103         continue;
2104       }
2105       // close the region -- close files and remove assignment
2106       HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
2107     }
2108   }
2109 
2110   private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
2111     KeeperException, InterruptedException {
2112     // If we are trying to fix the errors
2113     if (shouldFixAssignments()) {
2114       errors.print(msg);
2115       undeployRegions(hbi);
2116       setShouldRerun();
2117       HRegionInfo hri = hbi.getHdfsHRI();
2118       if (hri == null) {
2119         hri = hbi.metaEntry;
2120       }
2121       HBaseFsckRepair.fixUnassigned(admin, hri);
2122       HBaseFsckRepair.waitUntilAssigned(admin, hri);
2123 
2124       // also assign replicas if needed (do it only when this call operates on a primary replica)
2125       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
2126       int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
2127       for (int i = 1; i < replicationCount; i++) {
2128         hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
2129         HbckInfo h = regionInfoMap.get(hri.getEncodedName());
2130         if (h != null) {
2131           undeployRegions(h);
2132           //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore
2133           //in consistency checks
2134           h.setSkipChecks(true);
2135         }
2136         HBaseFsckRepair.fixUnassigned(admin, hri);
2137         HBaseFsckRepair.waitUntilAssigned(admin, hri);
2138       }
2139 
2140     }
2141   }
2142 
2143   /**
2144    * Check a single region for consistency and correct deployment.
2145    */
2146   private void checkRegionConsistency(final String key, final HbckInfo hbi)
2147   throws IOException, KeeperException, InterruptedException {
2148 
2149 	if (hbi.isSkipChecks()) return;
2150 	String descriptiveName = hbi.toString();
2151     boolean inMeta = hbi.metaEntry != null;
2152     // In case not checking HDFS, assume the region is on HDFS
2153     boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
2154     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
2155     boolean isDeployed = !hbi.deployedOn.isEmpty();
2156     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
2157     boolean deploymentMatchesMeta =
2158       hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
2159       hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
2160     boolean splitParent =
2161       (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
2162     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
2163     boolean recentlyModified = inHdfs &&
2164       hbi.getModTime() + timelag > EnvironmentEdgeManager.currentTime();
2165 
2166     // ========== First the healthy cases =============
2167     if (hbi.containsOnlyHdfsEdits()) {
2168       return;
2169     }
2170     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
2171       return;
2172     } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
2173       LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
2174         "tabled that is not deployed");
2175       return;
2176     } else if (recentlyModified) {
2177       LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
2178       return;
2179     }
2180     // ========== Cases where the region is not in hbase:meta =============
2181     else if (!inMeta && !inHdfs && !isDeployed) {
2182       // We shouldn't have record of this region at all then!
2183       assert false : "Entry for region with no data";
2184     } else if (!inMeta && !inHdfs && isDeployed) {
2185       errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
2186           + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
2187           "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2188       if (shouldFixAssignments()) {
2189         undeployRegions(hbi);
2190       }
2191 
2192     } else if (!inMeta && inHdfs && !isDeployed) {
2193       if (hbi.isMerged()) {
2194         // This region has already been merged, the remaining hdfs file will be
2195         // cleaned by CatalogJanitor later
2196         hbi.setSkipChecks(true);
2197         LOG.info("Region " + descriptiveName
2198             + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
2199         return;
2200       }
2201       errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
2202           + descriptiveName + " on HDFS, but not listed in hbase:meta " +
2203           "or deployed on any region server");
2204       // restore region consistency of an adopted orphan
2205       if (shouldFixMeta()) {
2206         if (!hbi.isHdfsRegioninfoPresent()) {
2207           LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
2208               +  " in table integrity repair phase if -fixHdfsOrphans was" +
2209               " used.");
2210           return;
2211         }
2212 
2213         HRegionInfo hri = hbi.getHdfsHRI();
2214         TableInfo tableInfo = tablesInfo.get(hri.getTable());
2215 
2216         for (HRegionInfo region : tableInfo.getRegionsFromMeta()) {
2217           if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0
2218               && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(),
2219                 hri.getEndKey()) >= 0)
2220               && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) {
2221             if(region.isSplit() || region.isOffline()) continue;
2222             Path regionDir = hbi.getHdfsRegionDir();
2223             FileSystem fs = regionDir.getFileSystem(getConf());
2224             List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir);
2225             for (Path familyDir : familyDirs) {
2226               List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);
2227               for (Path referenceFilePath : referenceFilePaths) {
2228                 Path parentRegionDir =
2229                     StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();
2230                 if (parentRegionDir.toString().endsWith(region.getEncodedName())) {
2231                   LOG.warn(hri + " start and stop keys are in the range of " + region
2232                       + ". The region might not be cleaned up from hdfs when region " + region
2233                       + " split failed. Hence deleting from hdfs.");
2234                   HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
2235                     regionDir.getParent(), hri);
2236                   return;
2237                 }
2238               }
2239             }
2240           }
2241         }
2242 
2243         LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
2244         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2245         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2246             admin.getClusterStatus().getServers(), numReplicas);
2247 
2248         tryAssignmentRepair(hbi, "Trying to reassign region...");
2249       }
2250 
2251     } else if (!inMeta && inHdfs && isDeployed) {
2252       errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
2253           + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2254       debugLsr(hbi.getHdfsRegionDir());
2255       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2256         // for replicas, this means that we should undeploy the region (we would have
2257         // gone over the primaries and fixed meta holes in first phase under
2258         // checkAndFixConsistency; we shouldn't get the condition !inMeta at
2259         // this stage unless unwanted replica)
2260         if (shouldFixAssignments()) {
2261           undeployRegionsForHbi(hbi);
2262         }
2263       }
2264       if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2265         if (!hbi.isHdfsRegioninfoPresent()) {
2266           LOG.error("This should have been repaired in table integrity repair phase");
2267           return;
2268         }
2269 
2270         LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
2271         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2272         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2273             admin.getClusterStatus().getServers(), numReplicas);
2274         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2275       }
2276 
2277     // ========== Cases where the region is in hbase:meta =============
2278     } else if (inMeta && inHdfs && !isDeployed && splitParent) {
2279       // check whether this is an actual error, or just transient state where parent
2280       // is not cleaned
2281       if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
2282         // check that split daughters are there
2283         HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
2284         HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
2285         if (infoA != null && infoB != null) {
2286           // we already processed or will process daughters. Move on, nothing to see here.
2287           hbi.setSkipChecks(true);
2288           return;
2289         }
2290       }
2291       errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2292           + descriptiveName + " is a split parent in META, in HDFS, "
2293           + "and not deployed on any region server. This could be transient, "
2294           + "consider to run the catalog janitor first!");
2295       if (shouldFixSplitParents()) {
2296         setShouldRerun();
2297         resetSplitParent(hbi);
2298       }
2299     } else if (inMeta && !inHdfs && !isDeployed) {
2300       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2301           + descriptiveName + " found in META, but not in HDFS "
2302           + "or deployed on any region server.");
2303       if (shouldFixMeta()) {
2304         deleteMetaRegion(hbi);
2305       }
2306     } else if (inMeta && !inHdfs && isDeployed) {
2307       errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2308           + " found in META, but not in HDFS, " +
2309           "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2310       // We treat HDFS as ground truth.  Any information in meta is transient
2311       // and equivalent data can be regenerated.  So, lets unassign and remove
2312       // these problems from META.
2313       if (shouldFixAssignments()) {
2314         errors.print("Trying to fix unassigned region...");
2315         undeployRegions(hbi);
2316       }
2317       if (shouldFixMeta()) {
2318         // wait for it to complete
2319         deleteMetaRegion(hbi);
2320       }
2321     } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2322       errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2323           + " not deployed on any region server.");
2324       tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2325     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2326       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2327           "Region " + descriptiveName + " should not be deployed according " +
2328           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2329       if (shouldFixAssignments()) {
2330         errors.print("Trying to close the region " + descriptiveName);
2331         setShouldRerun();
2332         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2333       }
2334     } else if (inMeta && inHdfs && isMultiplyDeployed) {
2335       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2336           + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
2337           + " but is multiply assigned to region servers " +
2338           Joiner.on(", ").join(hbi.deployedOn));
2339       // If we are trying to fix the errors
2340       if (shouldFixAssignments()) {
2341         errors.print("Trying to fix assignment error...");
2342         setShouldRerun();
2343         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2344       }
2345     } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2346       errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2347           + descriptiveName + " listed in hbase:meta on region server " +
2348           hbi.metaEntry.regionServer + " but found on region server " +
2349           hbi.deployedOn.get(0));
2350       // If we are trying to fix the errors
2351       if (shouldFixAssignments()) {
2352         errors.print("Trying to fix assignment error...");
2353         setShouldRerun();
2354         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2355         HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2356       }
2357     } else {
2358       errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2359           " is in an unforeseen state:" +
2360           " inMeta=" + inMeta +
2361           " inHdfs=" + inHdfs +
2362           " isDeployed=" + isDeployed +
2363           " isMultiplyDeployed=" + isMultiplyDeployed +
2364           " deploymentMatchesMeta=" + deploymentMatchesMeta +
2365           " shouldBeDeployed=" + shouldBeDeployed);
2366     }
2367   }
2368 
2369   /**
2370    * Checks tables integrity. Goes over all regions and scans the tables.
2371    * Collects all the pieces for each table and checks if there are missing,
2372    * repeated or overlapping ones.
2373    * @throws IOException
2374    */
2375   SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2376     tablesInfo = new TreeMap<TableName,TableInfo> ();
2377     LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2378     for (HbckInfo hbi : regionInfoMap.values()) {
2379       // Check only valid, working regions
2380       if (hbi.metaEntry == null) {
2381         // this assumes that consistency check has run loadMetaEntry
2382         Path p = hbi.getHdfsRegionDir();
2383         if (p == null) {
2384           errors.report("No regioninfo in Meta or HDFS. " + hbi);
2385         }
2386 
2387         // TODO test.
2388         continue;
2389       }
2390       if (hbi.metaEntry.regionServer == null) {
2391         errors.detail("Skipping region because no region server: " + hbi);
2392         continue;
2393       }
2394       if (hbi.metaEntry.isOffline()) {
2395         errors.detail("Skipping region because it is offline: " + hbi);
2396         continue;
2397       }
2398       if (hbi.containsOnlyHdfsEdits()) {
2399         errors.detail("Skipping region because it only contains edits" + hbi);
2400         continue;
2401       }
2402 
2403       // Missing regionDir or over-deployment is checked elsewhere. Include
2404       // these cases in modTInfo, so we can evaluate those regions as part of
2405       // the region chain in META
2406       //if (hbi.foundRegionDir == null) continue;
2407       //if (hbi.deployedOn.size() != 1) continue;
2408       if (hbi.deployedOn.size() == 0) continue;
2409 
2410       // We should be safe here
2411       TableName tableName = hbi.metaEntry.getTable();
2412       TableInfo modTInfo = tablesInfo.get(tableName);
2413       if (modTInfo == null) {
2414         modTInfo = new TableInfo(tableName);
2415       }
2416       for (ServerName server : hbi.deployedOn) {
2417         modTInfo.addServer(server);
2418       }
2419 
2420       if (!hbi.isSkipChecks()) {
2421         modTInfo.addRegionInfo(hbi);
2422       }
2423 
2424       tablesInfo.put(tableName, modTInfo);
2425     }
2426 
2427     loadTableInfosForTablesWithNoRegion();
2428 
2429     logParallelMerge();
2430     for (TableInfo tInfo : tablesInfo.values()) {
2431       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2432       if (!tInfo.checkRegionChain(handler)) {
2433         errors.report("Found inconsistency in table " + tInfo.getName());
2434       }
2435     }
2436     return tablesInfo;
2437   }
2438 
2439   /** Loads table info's for tables that may not have been included, since there are no
2440    * regions reported for the table, but table dir is there in hdfs
2441    */
2442   private void loadTableInfosForTablesWithNoRegion() throws IOException {
2443     Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2444     for (HTableDescriptor htd : allTables.values()) {
2445       if (checkMetaOnly && !htd.isMetaTable()) {
2446         continue;
2447       }
2448 
2449       TableName tableName = htd.getTableName();
2450       if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2451         TableInfo tableInfo = new TableInfo(tableName);
2452         tableInfo.htds.add(htd);
2453         tablesInfo.put(htd.getTableName(), tableInfo);
2454       }
2455     }
2456   }
2457 
2458   /**
2459    * Merge hdfs data by moving from contained HbckInfo into targetRegionDir.
2460    * @return number of file move fixes done to merge regions.
2461    */
2462   public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2463     int fileMoves = 0;
2464     String thread = Thread.currentThread().getName();
2465     LOG.debug("[" + thread + "] Contained region dir after close and pause");
2466     debugLsr(contained.getHdfsRegionDir());
2467 
2468     // rename the contained into the container.
2469     FileSystem fs = targetRegionDir.getFileSystem(getConf());
2470     FileStatus[] dirs = null;
2471     try {
2472       dirs = fs.listStatus(contained.getHdfsRegionDir());
2473     } catch (FileNotFoundException fnfe) {
2474       // region we are attempting to merge in is not present!  Since this is a merge, there is
2475       // no harm skipping this region if it does not exist.
2476       if (!fs.exists(contained.getHdfsRegionDir())) {
2477         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2478             + " is missing. Assuming already sidelined or moved.");
2479       } else {
2480         sidelineRegionDir(fs, contained);
2481       }
2482       return fileMoves;
2483     }
2484 
2485     if (dirs == null) {
2486       if (!fs.exists(contained.getHdfsRegionDir())) {
2487         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2488             + " already sidelined.");
2489       } else {
2490         sidelineRegionDir(fs, contained);
2491       }
2492       return fileMoves;
2493     }
2494 
2495     for (FileStatus cf : dirs) {
2496       Path src = cf.getPath();
2497       Path dst =  new Path(targetRegionDir, src.getName());
2498 
2499       if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2500         // do not copy the old .regioninfo file.
2501         continue;
2502       }
2503 
2504       if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2505         // do not copy the .oldlogs files
2506         continue;
2507       }
2508 
2509       LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2510       // FileSystem.rename is inconsistent with directories -- if the
2511       // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2512       // it moves the src into the dst dir resulting in (foo/a/b).  If
2513       // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2514       for (FileStatus hfile : fs.listStatus(src)) {
2515         boolean success = fs.rename(hfile.getPath(), dst);
2516         if (success) {
2517           fileMoves++;
2518         }
2519       }
2520       LOG.debug("[" + thread + "] Sideline directory contents:");
2521       debugLsr(targetRegionDir);
2522     }
2523 
2524     // if all success.
2525     sidelineRegionDir(fs, contained);
2526     LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2527         getSidelineDir());
2528     debugLsr(contained.getHdfsRegionDir());
2529 
2530     return fileMoves;
2531   }
2532 
2533 
2534   static class WorkItemOverlapMerge implements Callable<Void> {
2535     private TableIntegrityErrorHandler handler;
2536     Collection<HbckInfo> overlapgroup;
2537 
2538     WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2539       this.handler = handler;
2540       this.overlapgroup = overlapgroup;
2541     }
2542 
2543     @Override
2544     public Void call() throws Exception {
2545       handler.handleOverlapGroup(overlapgroup);
2546       return null;
2547     }
2548   };
2549 
2550 
2551   /**
2552    * Maintain information about a particular table.
2553    */
2554   public class TableInfo {
2555     TableName tableName;
2556     TreeSet <ServerName> deployedOn;
2557 
2558     // backwards regions
2559     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2560 
2561     // sidelined big overlapped regions
2562     final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2563 
2564     // region split calculator
2565     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2566 
2567     // Histogram of different HTableDescriptors found.  Ideally there is only one!
2568     final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2569 
2570     // key = start split, values = set of splits in problem group
2571     final Multimap<byte[], HbckInfo> overlapGroups =
2572       TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2573 
2574     // list of regions derived from meta entries.
2575     private ImmutableList<HRegionInfo> regionsFromMeta = null;
2576 
2577     TableInfo(TableName name) {
2578       this.tableName = name;
2579       deployedOn = new TreeSet <ServerName>();
2580     }
2581 
2582     /**
2583      * @return descriptor common to all regions.  null if are none or multiple!
2584      */
2585     private HTableDescriptor getHTD() {
2586       if (htds.size() == 1) {
2587         return (HTableDescriptor)htds.toArray()[0];
2588       } else {
2589         LOG.error("None/Multiple table descriptors found for table '"
2590           + tableName + "' regions: " + htds);
2591       }
2592       return null;
2593     }
2594 
2595     public void addRegionInfo(HbckInfo hir) {
2596       if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2597         // end key is absolute end key, just add it.
2598         // ignore replicas other than primary for these checks
2599         if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2600         return;
2601       }
2602 
2603       // if not the absolute end key, check for cycle
2604       if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2605         errors.reportError(
2606             ERROR_CODE.REGION_CYCLE,
2607             String.format("The endkey for this region comes before the "
2608                 + "startkey, startkey=%s, endkey=%s",
2609                 Bytes.toStringBinary(hir.getStartKey()),
2610                 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2611         backwards.add(hir);
2612         return;
2613       }
2614 
2615       // main case, add to split calculator
2616       // ignore replicas other than primary for these checks
2617       if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2618     }
2619 
2620     public void addServer(ServerName server) {
2621       this.deployedOn.add(server);
2622     }
2623 
2624     public TableName getName() {
2625       return tableName;
2626     }
2627 
2628     public int getNumRegions() {
2629       return sc.getStarts().size() + backwards.size();
2630     }
2631 
2632     public synchronized ImmutableList<HRegionInfo> getRegionsFromMeta() {
2633       // lazy loaded, synchronized to ensure a single load
2634       if (regionsFromMeta == null) {
2635         List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
2636         for (HbckInfo h : HBaseFsck.this.regionInfoMap.values()) {
2637           if (tableName.equals(h.getTableName())) {
2638             if (h.metaEntry != null) {
2639               regions.add((HRegionInfo) h.metaEntry);
2640             }
2641           }
2642         }
2643         regionsFromMeta = Ordering.natural().immutableSortedCopy(regions);
2644       }
2645       
2646       return regionsFromMeta;
2647     }
2648     
2649 
2650       private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2651       ErrorReporter errors;
2652 
2653       IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2654         this.errors = errors;
2655         setTableInfo(ti);
2656       }
2657 
2658       @Override
2659       public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2660         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2661             "First region should start with an empty key.  You need to "
2662             + " create a new region and regioninfo in HDFS to plug the hole.",
2663             getTableInfo(), hi);
2664       }
2665 
2666       @Override
2667       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2668         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2669             "Last region should end with an empty key. You need to "
2670                 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2671       }
2672 
2673       @Override
2674       public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2675         errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2676             "Region has the same start and end key.", getTableInfo(), hi);
2677       }
2678 
2679       @Override
2680       public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2681         byte[] key = r1.getStartKey();
2682         // dup start key
2683         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2684             "Multiple regions have the same startkey: "
2685             + Bytes.toStringBinary(key), getTableInfo(), r1);
2686         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2687             "Multiple regions have the same startkey: "
2688             + Bytes.toStringBinary(key), getTableInfo(), r2);
2689       }
2690 
2691       @Override
2692       public void handleSplit(HbckInfo r1, HbckInfo r2) throws IOException{
2693         byte[] key = r1.getStartKey();
2694         // dup start key
2695         errors.reportError(ERROR_CODE.DUPE_ENDKEYS,
2696           "Multiple regions have the same regionID: "
2697             + Bytes.toStringBinary(key), getTableInfo(), r1);
2698         errors.reportError(ERROR_CODE.DUPE_ENDKEYS,
2699           "Multiple regions have the same regionID: "
2700             + Bytes.toStringBinary(key), getTableInfo(), r2);
2701       }
2702 
2703       @Override
2704       public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2705         errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2706             "There is an overlap in the region chain.",
2707             getTableInfo(), hi1, hi2);
2708       }
2709 
2710       @Override
2711       public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2712         errors.reportError(
2713             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2714             "There is a hole in the region chain between "
2715                 + Bytes.toStringBinary(holeStart) + " and "
2716                 + Bytes.toStringBinary(holeStop)
2717                 + ".  You need to create a new .regioninfo and region "
2718                 + "dir in hdfs to plug the hole.");
2719       }
2720     };
2721 
2722     /**
2723      * This handler fixes integrity errors from hdfs information.  There are
2724      * basically three classes of integrity problems 1) holes, 2) overlaps, and
2725      * 3) invalid regions.
2726      *
2727      * This class overrides methods that fix holes and the overlap group case.
2728      * Individual cases of particular overlaps are handled by the general
2729      * overlap group merge repair case.
2730      *
2731      * If hbase is online, this forces regions offline before doing merge
2732      * operations.
2733      */
2734     private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2735       Configuration conf;
2736 
2737       boolean fixOverlaps = true;
2738 
2739       HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2740           boolean fixHoles, boolean fixOverlaps) {
2741         super(ti, errors);
2742         this.conf = conf;
2743         this.fixOverlaps = fixOverlaps;
2744         // TODO properly use fixHoles
2745       }
2746 
2747       /**
2748        * This is a special case hole -- when the first region of a table is
2749        * missing from META, HBase doesn't acknowledge the existance of the
2750        * table.
2751        */
2752       @Override
2753       public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2754         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2755             "First region should start with an empty key.  Creating a new " +
2756             "region and regioninfo in HDFS to plug the hole.",
2757             getTableInfo(), next);
2758         HTableDescriptor htd = getTableInfo().getHTD();
2759         // from special EMPTY_START_ROW to next region's startKey
2760         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2761             HConstants.EMPTY_START_ROW, next.getStartKey());
2762 
2763         // TODO test
2764         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2765         LOG.info("Table region start key was not empty.  Created new empty region: "
2766             + newRegion + " " +region);
2767         fixes++;
2768       }
2769 
2770       @Override
2771       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2772         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2773             "Last region should end with an empty key.  Creating a new "
2774                 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2775         HTableDescriptor htd = getTableInfo().getHTD();
2776         // from curEndKey to EMPTY_START_ROW
2777         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2778             HConstants.EMPTY_START_ROW);
2779 
2780         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2781         LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
2782             + " " + region);
2783         fixes++;
2784       }
2785 
2786       /**
2787        * There is a hole in the hdfs regions that violates the table integrity
2788        * rules.  Create a new empty region that patches the hole.
2789        */
2790       @Override
2791       public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2792         errors.reportError(
2793             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2794             "There is a hole in the region chain between "
2795                 + Bytes.toStringBinary(holeStartKey) + " and "
2796                 + Bytes.toStringBinary(holeStopKey)
2797                 + ".  Creating a new regioninfo and region "
2798                 + "dir in hdfs to plug the hole.");
2799         HTableDescriptor htd = getTableInfo().getHTD();
2800         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2801         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2802         LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2803         fixes++;
2804       }
2805 
2806       /**
2807        * This takes set of overlapping regions and merges them into a single
2808        * region.  This covers cases like degenerate regions, shared start key,
2809        * general overlaps, duplicate ranges, and partial overlapping regions.
2810        *
2811        * Cases:
2812        * - Clean regions that overlap
2813        * - Only .oldlogs regions (can't find start/stop range, or figure out)
2814        *
2815        * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
2816        */
2817       @Override
2818       public void handleOverlapGroup(Collection<HbckInfo> overlap)
2819           throws IOException {
2820         Preconditions.checkNotNull(overlap);
2821         Preconditions.checkArgument(overlap.size() >0);
2822 
2823         if (!this.fixOverlaps) {
2824           LOG.warn("Not attempting to repair overlaps.");
2825           return;
2826         }
2827 
2828         if (overlap.size() > maxMerge) {
2829           LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2830             "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2831           if (sidelineBigOverlaps) {
2832             // we only sideline big overlapped groups that exceeds the max number of regions to merge
2833             sidelineBigOverlaps(overlap);
2834           }
2835           return;
2836         }
2837         if (shouldRemoveParents()) {
2838           removeParentsAndFixSplits(overlap);
2839         }
2840         mergeOverlaps(overlap);
2841       }
2842 
2843       void removeParentsAndFixSplits(Collection<HbckInfo> overlap) throws IOException {
2844         Pair<byte[], byte[]> range = null;
2845         HbckInfo parent = null;
2846         HbckInfo daughterA = null;
2847         HbckInfo daughterB = null;
2848         Collection<HbckInfo> daughters = new ArrayList<HbckInfo>(overlap);
2849 
2850         String thread = Thread.currentThread().getName();
2851         LOG.info("== [" + thread + "] Attempting fix splits in overlap state.");
2852 
2853         // we only can handle a single split per group at the time
2854         if (overlap.size() > 3) {
2855           LOG.info("Too many overlaps were found on this group, falling back to regular merge.");
2856           return;
2857         }
2858 
2859         for (HbckInfo hi : overlap) {
2860           if (range == null) {
2861             range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2862           } else {
2863             if (RegionSplitCalculator.BYTES_COMPARATOR
2864               .compare(hi.getStartKey(), range.getFirst()) < 0) {
2865               range.setFirst(hi.getStartKey());
2866             }
2867             if (RegionSplitCalculator.BYTES_COMPARATOR
2868               .compare(hi.getEndKey(), range.getSecond()) > 0) {
2869               range.setSecond(hi.getEndKey());
2870             }
2871           }
2872         }
2873 
2874         LOG.info("This group range is [" + Bytes.toStringBinary(range.getFirst()) + ", "
2875           + Bytes.toStringBinary(range.getSecond()) + "]");
2876 
2877         // attempt to find a possible parent for the edge case of a split
2878         for (HbckInfo hi : overlap) {
2879           if (Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0
2880             && Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0) {
2881             LOG.info("This is a parent for this group: " + hi.toString());
2882             parent = hi;
2883           }
2884         }
2885 
2886         // Remove parent regions from daughters collection
2887         if (parent != null) {
2888           daughters.remove(parent);
2889         }
2890 
2891         // Lets verify that daughters share the regionID at split time and they
2892         // were created after the parent
2893         for (HbckInfo hi : daughters) {
2894           if (Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0) {
2895             if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) {
2896               daughterA = hi;
2897             }
2898           }
2899           if (Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0) {
2900             if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) {
2901               daughterB = hi;
2902             }
2903           }
2904         }
2905 
2906         // daughters must share the same regionID and we should have a parent too
2907         if (daughterA.getHdfsHRI().getRegionId() != daughterB.getHdfsHRI().getRegionId() || parent == null)
2908           return;
2909 
2910         FileSystem fs = FileSystem.get(conf);
2911         LOG.info("Found parent: " + parent.getRegionNameAsString());
2912         LOG.info("Found potential daughter a: " + daughterA.getRegionNameAsString());
2913         LOG.info("Found potential daughter b: " + daughterB.getRegionNameAsString());
2914         LOG.info("Trying to fix parent in overlap by removing the parent.");
2915         try {
2916           closeRegion(parent);
2917         } catch (IOException ioe) {
2918           LOG.warn("Parent region could not be closed, continuing with regular merge...", ioe);
2919           return;
2920         } catch (InterruptedException ie) {
2921           LOG.warn("Parent region could not be closed, continuing with regular merge...", ie);
2922           return;
2923         }
2924 
2925         try {
2926           offline(parent.getRegionName());
2927         } catch (IOException ioe) {
2928           LOG.warn("Unable to offline parent region: " + parent.getRegionNameAsString()
2929             + ".  Just continuing with regular merge... ", ioe);
2930           return;
2931         }
2932 
2933         try {
2934           HBaseFsckRepair.removeParentInMeta(conf, parent.getHdfsHRI());
2935         } catch (IOException ioe) {
2936           LOG.warn("Unable to remove parent region in META: " + parent.getRegionNameAsString()
2937             + ".  Just continuing with regular merge... ", ioe);
2938           return;
2939         }
2940 
2941         sidelineRegionDir(fs, parent);
2942         LOG.info("[" + thread + "] Sidelined parent region dir "+ parent.getHdfsRegionDir() + " into " +
2943           getSidelineDir());
2944         debugLsr(parent.getHdfsRegionDir());
2945 
2946         // Make sure we don't have the parents and daughters around
2947         overlap.remove(parent);
2948         overlap.remove(daughterA);
2949         overlap.remove(daughterB);
2950 
2951         LOG.info("Done fixing split.");
2952 
2953       }
2954 
2955       void mergeOverlaps(Collection<HbckInfo> overlap)
2956           throws IOException {
2957         String thread = Thread.currentThread().getName();
2958         LOG.info("== [" + thread + "] Merging regions into one region: "
2959           + Joiner.on(",").join(overlap));
2960         // get the min / max range and close all concerned regions
2961         Pair<byte[], byte[]> range = null;
2962         for (HbckInfo hi : overlap) {
2963           if (range == null) {
2964             range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2965           } else {
2966             if (RegionSplitCalculator.BYTES_COMPARATOR
2967                 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2968               range.setFirst(hi.getStartKey());
2969             }
2970             if (RegionSplitCalculator.BYTES_COMPARATOR
2971                 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2972               range.setSecond(hi.getEndKey());
2973             }
2974           }
2975           // need to close files so delete can happen.
2976           LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
2977           LOG.debug("[" + thread + "] Contained region dir before close");
2978           debugLsr(hi.getHdfsRegionDir());
2979           try {
2980             LOG.info("[" + thread + "] Closing region: " + hi);
2981             closeRegion(hi);
2982           } catch (IOException ioe) {
2983             LOG.warn("[" + thread + "] Was unable to close region " + hi
2984               + ".  Just continuing... ", ioe);
2985           } catch (InterruptedException e) {
2986             LOG.warn("[" + thread + "] Was unable to close region " + hi
2987               + ".  Just continuing... ", e);
2988           }
2989 
2990           try {
2991             LOG.info("[" + thread + "] Offlining region: " + hi);
2992             offline(hi.getRegionName());
2993           } catch (IOException ioe) {
2994             LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2995               + ".  Just continuing... ", ioe);
2996           }
2997         }
2998 
2999         // create new empty container region.
3000         HTableDescriptor htd = getTableInfo().getHTD();
3001         // from start key to end Key
3002         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
3003             range.getSecond());
3004         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
3005         LOG.info("[" + thread + "] Created new empty container region: " +
3006             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
3007         debugLsr(region.getRegionFileSystem().getRegionDir());
3008 
3009         // all target regions are closed, should be able to safely cleanup.
3010         boolean didFix= false;
3011         Path target = region.getRegionFileSystem().getRegionDir();
3012         for (HbckInfo contained : overlap) {
3013           LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
3014           int merges = mergeRegionDirs(target, contained);
3015           if (merges > 0) {
3016             didFix = true;
3017           }
3018         }
3019         if (didFix) {
3020           fixes++;
3021         }
3022       }
3023 
3024       /**
3025        * Sideline some regions in a big overlap group so that it
3026        * will have fewer regions, and it is easier to merge them later on.
3027        *
3028        * @param bigOverlap the overlapped group with regions more than maxMerge
3029        * @throws IOException
3030        */
3031       void sidelineBigOverlaps(
3032           Collection<HbckInfo> bigOverlap) throws IOException {
3033         int overlapsToSideline = bigOverlap.size() - maxMerge;
3034         if (overlapsToSideline > maxOverlapsToSideline) {
3035           overlapsToSideline = maxOverlapsToSideline;
3036         }
3037         List<HbckInfo> regionsToSideline =
3038           RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
3039         FileSystem fs = FileSystem.get(conf);
3040         for (HbckInfo regionToSideline: regionsToSideline) {
3041           try {
3042             LOG.info("Closing region: " + regionToSideline);
3043             closeRegion(regionToSideline);
3044           } catch (IOException ioe) {
3045             LOG.warn("Was unable to close region " + regionToSideline
3046               + ".  Just continuing... ", ioe);
3047           } catch (InterruptedException e) {
3048             LOG.warn("Was unable to close region " + regionToSideline
3049               + ".  Just continuing... ", e);
3050           }
3051 
3052           try {
3053             LOG.info("Offlining region: " + regionToSideline);
3054             offline(regionToSideline.getRegionName());
3055           } catch (IOException ioe) {
3056             LOG.warn("Unable to offline region from master: " + regionToSideline
3057               + ".  Just continuing... ", ioe);
3058           }
3059 
3060           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
3061           Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
3062           if (sidelineRegionDir != null) {
3063             sidelinedRegions.put(sidelineRegionDir, regionToSideline);
3064             LOG.info("After sidelined big overlapped region: "
3065               + regionToSideline.getRegionNameAsString()
3066               + " to " + sidelineRegionDir.toString());
3067             fixes++;
3068           }
3069         }
3070       }
3071     }
3072 
3073     /**
3074      * Check the region chain (from META) of this table.  We are looking for
3075      * holes, overlaps, and cycles.
3076      * @return false if there are errors
3077      * @throws IOException
3078      */
3079     public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
3080       // When table is disabled no need to check for the region chain. Some of the regions
3081       // accidently if deployed, this below code might report some issues like missing start
3082       // or end regions or region hole in chain and may try to fix which is unwanted.
3083       if (disabledTables.contains(this.tableName)) {
3084         return true;
3085       }
3086       int originalErrorsCount = errors.getErrorList().size();
3087       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
3088       SortedSet<byte[]> splits = sc.getSplits();
3089 
3090       byte[] prevKey = null;
3091       byte[] problemKey = null;
3092 
3093       if (splits.size() == 0) {
3094         // no region for this table
3095         handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
3096       }
3097 
3098       for (byte[] key : splits) {
3099         Collection<HbckInfo> ranges = regions.get(key);
3100         if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
3101           for (HbckInfo rng : ranges) {
3102             handler.handleRegionStartKeyNotEmpty(rng);
3103           }
3104         }
3105 
3106         // check for degenerate ranges
3107         for (HbckInfo rng : ranges) {
3108           // special endkey case converts '' to null
3109           byte[] endKey = rng.getEndKey();
3110           endKey = (endKey.length == 0) ? null : endKey;
3111           if (Bytes.equals(rng.getStartKey(),endKey)) {
3112             handler.handleDegenerateRegion(rng);
3113           }
3114         }
3115 
3116         if (ranges.size() == 1) {
3117           // this split key is ok -- no overlap, not a hole.
3118           if (problemKey != null) {
3119             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3120           }
3121           problemKey = null; // fell through, no more problem.
3122         } else if (ranges.size() > 1) {
3123           // set the new problem key group name, if already have problem key, just
3124           // keep using it.
3125           if (problemKey == null) {
3126             // only for overlap regions.
3127             LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
3128             problemKey = key;
3129           }
3130           overlapGroups.putAll(problemKey, ranges);
3131 
3132           // record errors
3133           ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
3134           //  this dumb and n^2 but this shouldn't happen often
3135           for (HbckInfo r1 : ranges) {
3136             if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3137             subRange.remove(r1);
3138             for (HbckInfo r2 : subRange) {
3139               if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
3140               // general case of same start key
3141               if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
3142                 handler.handleDuplicateStartKeys(r1,r2);
3143               } else if (Bytes.compareTo(r1.getEndKey(), r2.getStartKey())==0 &&
3144                 r1.getHdfsHRI().getRegionId() == r2.getHdfsHRI().getRegionId()) {
3145                 LOG.info("this is a split, log to splits");
3146                 handler.handleSplit(r1, r2);
3147               } else {
3148                 // overlap
3149                 handler.handleOverlapInRegionChain(r1, r2);
3150               }
3151             }
3152           }
3153 
3154         } else if (ranges.size() == 0) {
3155           if (problemKey != null) {
3156             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
3157           }
3158           problemKey = null;
3159 
3160           byte[] holeStopKey = sc.getSplits().higher(key);
3161           // if higher key is null we reached the top.
3162           if (holeStopKey != null) {
3163             // hole
3164             handler.handleHoleInRegionChain(key, holeStopKey);
3165           }
3166         }
3167         prevKey = key;
3168       }
3169 
3170       // When the last region of a table is proper and having an empty end key, 'prevKey'
3171       // will be null.
3172       if (prevKey != null) {
3173         handler.handleRegionEndKeyNotEmpty(prevKey);
3174       }
3175 
3176       // TODO fold this into the TableIntegrityHandler
3177       if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
3178         boolean ok = handleOverlapsParallel(handler, prevKey);
3179         if (!ok) {
3180           return false;
3181         }
3182       } else {
3183         for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3184           handler.handleOverlapGroup(overlap);
3185         }
3186       }
3187 
3188       if (details) {
3189         // do full region split map dump
3190         errors.print("---- Table '"  +  this.tableName
3191             + "': region split map");
3192         dump(splits, regions);
3193         errors.print("---- Table '"  +  this.tableName
3194             + "': overlap groups");
3195         dumpOverlapProblems(overlapGroups);
3196         errors.print("There are " + overlapGroups.keySet().size()
3197             + " overlap groups with " + overlapGroups.size()
3198             + " overlapping regions");
3199       }
3200       if (!sidelinedRegions.isEmpty()) {
3201         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
3202         errors.print("---- Table '"  +  this.tableName
3203             + "': sidelined big overlapped regions");
3204         dumpSidelinedRegions(sidelinedRegions);
3205       }
3206       return errors.getErrorList().size() == originalErrorsCount;
3207     }
3208 
3209     private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
3210         throws IOException {
3211       // we parallelize overlap handler for the case we have lots of groups to fix.  We can
3212       // safely assume each group is independent.
3213       List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
3214       List<Future<Void>> rets;
3215       for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
3216         //
3217         merges.add(new WorkItemOverlapMerge(overlap, handler));
3218       }
3219       try {
3220         rets = executor.invokeAll(merges);
3221       } catch (InterruptedException e) {
3222         LOG.error("Overlap merges were interrupted", e);
3223         return false;
3224       }
3225       for(int i=0; i<merges.size(); i++) {
3226         WorkItemOverlapMerge work = merges.get(i);
3227         Future<Void> f = rets.get(i);
3228         try {
3229           f.get();
3230         } catch(ExecutionException e) {
3231           LOG.warn("Failed to merge overlap group" + work, e.getCause());
3232         } catch (InterruptedException e) {
3233           LOG.error("Waiting for overlap merges was interrupted", e);
3234           return false;
3235         }
3236       }
3237       return true;
3238     }
3239 
3240     /**
3241      * This dumps data in a visually reasonable way for visual debugging
3242      *
3243      * @param splits
3244      * @param regions
3245      */
3246     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
3247       // we display this way because the last end key should be displayed as well.
3248       StringBuilder sb = new StringBuilder();
3249       for (byte[] k : splits) {
3250         sb.setLength(0); // clear out existing buffer, if any.
3251         sb.append(Bytes.toStringBinary(k) + ":\t");
3252         for (HbckInfo r : regions.get(k)) {
3253           sb.append("[ "+ r.toString() + ", "
3254               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
3255         }
3256         errors.print(sb.toString());
3257       }
3258     }
3259   }
3260 
3261   public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
3262     // we display this way because the last end key should be displayed as
3263     // well.
3264     for (byte[] k : regions.keySet()) {
3265       errors.print(Bytes.toStringBinary(k) + ":");
3266       for (HbckInfo r : regions.get(k)) {
3267         errors.print("[ " + r.toString() + ", "
3268             + Bytes.toStringBinary(r.getEndKey()) + "]");
3269       }
3270       errors.print("----");
3271     }
3272   }
3273 
3274   public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
3275     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
3276       TableName tableName = entry.getValue().getTableName();
3277       Path path = entry.getKey();
3278       errors.print("This sidelined region dir should be bulk loaded: "
3279         + path.toString());
3280       errors.print("Bulk load command looks like: "
3281         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
3282         + path.toUri().getPath() + " "+ tableName);
3283     }
3284   }
3285 
3286   public Multimap<byte[], HbckInfo> getOverlapGroups(
3287       TableName table) {
3288     TableInfo ti = tablesInfo.get(table);
3289     return ti.overlapGroups;
3290   }
3291 
3292   /**
3293    * Return a list of user-space table names whose metadata have not been
3294    * modified in the last few milliseconds specified by timelag
3295    * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
3296    * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
3297    * milliseconds specified by timelag, then the table is a candidate to be returned.
3298    * @return tables that have not been modified recently
3299    * @throws IOException if an error is encountered
3300    */
3301   HTableDescriptor[] getTables(AtomicInteger numSkipped) {
3302     List<TableName> tableNames = new ArrayList<TableName>();
3303     long now = EnvironmentEdgeManager.currentTime();
3304 
3305     for (HbckInfo hbi : regionInfoMap.values()) {
3306       MetaEntry info = hbi.metaEntry;
3307 
3308       // if the start key is zero, then we have found the first region of a table.
3309       // pick only those tables that were not modified in the last few milliseconds.
3310       if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
3311         if (info.modTime + timelag < now) {
3312           tableNames.add(info.getTable());
3313         } else {
3314           numSkipped.incrementAndGet(); // one more in-flux table
3315         }
3316       }
3317     }
3318     return getHTableDescriptors(tableNames);
3319   }
3320 
3321   HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
3322     HTableDescriptor[] htd = new HTableDescriptor[0];
3323     Admin admin = null;
3324     try {
3325       LOG.info("getHTableDescriptors == tableNames => " + tableNames);
3326       admin = new HBaseAdmin(getConf());
3327       htd = admin.getTableDescriptorsByTableName(tableNames);
3328     } catch (IOException e) {
3329       LOG.debug("Exception getting table descriptors", e);
3330     } finally {
3331       if (admin != null) {
3332         try {
3333           admin.close();
3334         } catch (IOException e) {
3335           LOG.debug("Exception closing HBaseAdmin", e);
3336         }
3337       }
3338     }
3339     return htd;
3340   }
3341 
3342   /**
3343    * Gets the entry in regionInfo corresponding to the the given encoded
3344    * region name. If the region has not been seen yet, a new entry is added
3345    * and returned.
3346    */
3347   private synchronized HbckInfo getOrCreateInfo(String name) {
3348     HbckInfo hbi = regionInfoMap.get(name);
3349     if (hbi == null) {
3350       hbi = new HbckInfo(null);
3351       regionInfoMap.put(name, hbi);
3352     }
3353     return hbi;
3354   }
3355 
3356   private void checkAndFixTableLocks() throws IOException {
3357     ZooKeeperWatcher zkw = createZooKeeperWatcher();
3358 
3359     try {
3360       TableLockChecker checker = new TableLockChecker(zkw, errors);
3361       checker.checkTableLocks();
3362 
3363       if (this.fixTableLocks) {
3364         checker.fixExpiredTableLocks();
3365       }
3366     } finally {
3367       zkw.close();
3368     }
3369   }
3370 
3371   /**
3372    * Check whether a orphaned table ZNode exists and fix it if requested.
3373    * @throws IOException
3374    * @throws KeeperException
3375    * @throws InterruptedException
3376    */
3377   private void checkAndFixOrphanedTableZNodes()
3378       throws IOException, KeeperException, InterruptedException {
3379     ZooKeeperWatcher zkw = createZooKeeperWatcher();
3380 
3381     try {
3382       Set<TableName> enablingTables = ZKTableStateClientSideReader.getEnablingTables(zkw);
3383       String msg;
3384       TableInfo tableInfo;
3385 
3386       for (TableName tableName : enablingTables) {
3387         // Check whether the table exists in hbase
3388         tableInfo = tablesInfo.get(tableName);
3389         if (tableInfo != null) {
3390           // Table exists.  This table state is in transit.  No problem for this table.
3391           continue;
3392         }
3393 
3394         msg = "Table " + tableName + " not found in hbase:meta. Orphaned table ZNode found.";
3395         LOG.warn(msg);
3396         orphanedTableZNodes.add(tableName);
3397         errors.reportError(ERROR_CODE.ORPHANED_ZK_TABLE_ENTRY, msg);
3398       }
3399 
3400       if (orphanedTableZNodes.size() > 0 && this.fixTableZNodes) {
3401         ZKTableStateManager zkTableStateMgr = new ZKTableStateManager(zkw);
3402 
3403         for (TableName tableName : orphanedTableZNodes) {
3404           try {
3405             // Set the table state to be disabled so that if we made mistake, we can trace
3406             // the history and figure it out.
3407             // Another choice is to call checkAndRemoveTableState() to delete the orphaned ZNode.
3408             // Both approaches works.
3409             zkTableStateMgr.setTableState(tableName, ZooKeeperProtos.Table.State.DISABLED);
3410           } catch (CoordinatedStateException e) {
3411             // This exception should not happen here
3412             LOG.error(
3413               "Got a CoordinatedStateException while fixing the ENABLING table znode " + tableName,
3414               e);
3415           }
3416         }
3417       }
3418     } finally {
3419       zkw.close();
3420     }
3421   }
3422 
3423   /**
3424     * Check values in regionInfo for hbase:meta
3425     * Check if zero or more than one regions with hbase:meta are found.
3426     * If there are inconsistencies (i.e. zero or more than one regions
3427     * pretend to be holding the hbase:meta) try to fix that and report an error.
3428     * @throws IOException from HBaseFsckRepair functions
3429     * @throws KeeperException
3430     * @throws InterruptedException
3431     */
3432   boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
3433     Map<Integer, HbckInfo> metaRegions = new HashMap<Integer, HbckInfo>();
3434     for (HbckInfo value : regionInfoMap.values()) {
3435       if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
3436         metaRegions.put(value.getReplicaId(), value);
3437       }
3438     }
3439     int metaReplication = admin.getTableDescriptor(TableName.META_TABLE_NAME)
3440         .getRegionReplication();
3441     boolean noProblem = true;
3442     // There will be always entries in regionInfoMap corresponding to hbase:meta & its replicas
3443     // Check the deployed servers. It should be exactly one server for each replica.
3444     for (int i = 0; i < metaReplication; i++) {
3445       HbckInfo metaHbckInfo = metaRegions.remove(i);
3446       List<ServerName> servers = new ArrayList<ServerName>();
3447       if (metaHbckInfo != null) {
3448         servers = metaHbckInfo.deployedOn;
3449       }
3450       if (servers.size() != 1) {
3451         noProblem = false;
3452         if (servers.size() == 0) {
3453           assignMetaReplica(i);
3454         } else if (servers.size() > 1) {
3455           errors
3456           .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId " +
3457                        metaHbckInfo.getReplicaId() + " is found on more than one region.");
3458           if (shouldFixAssignments()) {
3459             errors.print("Trying to fix a problem with hbase:meta, replicaId " +
3460                          metaHbckInfo.getReplicaId() +"..");
3461             setShouldRerun();
3462             // try fix it (treat is a dupe assignment)
3463             HBaseFsckRepair.fixMultiAssignment(connection, metaHbckInfo.metaEntry, servers);
3464           }
3465         }
3466       }
3467     }
3468     // unassign whatever is remaining in metaRegions. They are excess replicas.
3469     for (Map.Entry<Integer, HbckInfo> entry : metaRegions.entrySet()) {
3470       noProblem = false;
3471       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
3472           "hbase:meta replicas are deployed in excess. Configured " + metaReplication +
3473           ", deployed " + metaRegions.size());
3474       if (shouldFixAssignments()) {
3475         errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() +
3476             " of hbase:meta..");
3477         setShouldRerun();
3478         unassignMetaReplica(entry.getValue());
3479       }
3480     }
3481     // if noProblem is false, rerun hbck with hopefully fixed META
3482     // if noProblem is true, no errors, so continue normally
3483     return noProblem;
3484   }
3485 
3486   private void unassignMetaReplica(HbckInfo hi) throws IOException, InterruptedException,
3487   KeeperException {
3488     undeployRegions(hi);
3489     ZooKeeperWatcher zkw = createZooKeeperWatcher();
3490     try {
3491       ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(hi.metaEntry.getReplicaId()));
3492     } finally {
3493       zkw.close();
3494     }
3495   }
3496 
3497   private void assignMetaReplica(int replicaId)
3498       throws IOException, KeeperException, InterruptedException {
3499     errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " +
3500         replicaId +" is not found on any region.");
3501     if (shouldFixAssignments()) {
3502       errors.print("Trying to fix a problem with hbase:meta..");
3503       setShouldRerun();
3504       // try to fix it (treat it as unassigned region)
3505       HRegionInfo h = RegionReplicaUtil.getRegionInfoForReplica(
3506           HRegionInfo.FIRST_META_REGIONINFO, replicaId);
3507       HBaseFsckRepair.fixUnassigned(admin, h);
3508       HBaseFsckRepair.waitUntilAssigned(admin, h);
3509     }
3510   }
3511 
3512   /**
3513    * Scan hbase:meta, adding all regions found to the regionInfo map.
3514    * @throws IOException if an error is encountered
3515    */
3516   boolean loadMetaEntries() throws IOException {
3517     MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
3518       int countRecord = 1;
3519 
3520       // comparator to sort KeyValues with latest modtime
3521       final Comparator<Cell> comp = new Comparator<Cell>() {
3522         @Override
3523         public int compare(Cell k1, Cell k2) {
3524           return Long.compare(k1.getTimestamp(), k2.getTimestamp());
3525         }
3526       };
3527 
3528       @Override
3529       public boolean processRow(Result result) throws IOException {
3530         try {
3531 
3532           // record the latest modification of this META record
3533           long ts =  Collections.max(result.listCells(), comp).getTimestamp();
3534           RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
3535           if (rl == null) {
3536             emptyRegionInfoQualifiers.add(result);
3537             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3538               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3539             return true;
3540           }
3541           ServerName sn = null;
3542           if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
3543               rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
3544             emptyRegionInfoQualifiers.add(result);
3545             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3546               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3547             return true;
3548           }
3549           HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
3550           if (!(isTableIncluded(hri.getTable())
3551               || hri.isMetaRegion())) {
3552             return true;
3553           }
3554           PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
3555           for (HRegionLocation h : rl.getRegionLocations()) {
3556             if (h == null || h.getRegionInfo() == null) {
3557               continue;
3558             }
3559             sn = h.getServerName();
3560             hri = h.getRegionInfo();
3561 
3562             MetaEntry m = null;
3563             if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
3564               m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
3565             } else {
3566               m = new MetaEntry(hri, sn, ts, null, null);
3567             }
3568             HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
3569             if (previous == null) {
3570               regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
3571             } else if (previous.metaEntry == null) {
3572               previous.metaEntry = m;
3573             } else {
3574               throw new IOException("Two entries in hbase:meta are same " + previous);
3575             }
3576           }
3577           PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
3578           for (HRegionInfo mergeRegion : new HRegionInfo[] {
3579               mergeRegions.getFirst(), mergeRegions.getSecond() }) {
3580             if (mergeRegion != null) {
3581               // This region is already been merged
3582               HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
3583               hbInfo.setMerged(true);
3584             }
3585           }
3586 
3587           // show proof of progress to the user, once for every 100 records.
3588           if (countRecord % 100 == 0) {
3589             errors.progress();
3590           }
3591           countRecord++;
3592           return true;
3593         } catch (RuntimeException e) {
3594           LOG.error("Result=" + result);
3595           throw e;
3596         }
3597       }
3598     };
3599     if (!checkMetaOnly) {
3600       // Scan hbase:meta to pick up user regions
3601       MetaScanner.metaScan(connection, visitor);
3602     }
3603 
3604     errors.print("");
3605     return true;
3606   }
3607 
3608   /**
3609    * Stores the regioninfo entries scanned from META
3610    */
3611   static class MetaEntry extends HRegionInfo {
3612     ServerName regionServer;   // server hosting this region
3613     long modTime;          // timestamp of most recent modification metadata
3614     HRegionInfo splitA, splitB; //split daughters
3615 
3616     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3617       this(rinfo, regionServer, modTime, null, null);
3618     }
3619 
3620     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3621         HRegionInfo splitA, HRegionInfo splitB) {
3622       super(rinfo);
3623       this.regionServer = regionServer;
3624       this.modTime = modTime;
3625       this.splitA = splitA;
3626       this.splitB = splitB;
3627     }
3628 
3629     @Override
3630     public boolean equals(Object o) {
3631       boolean superEq = super.equals(o);
3632       if (!superEq) {
3633         return superEq;
3634       }
3635 
3636       MetaEntry me = (MetaEntry) o;
3637       if (!regionServer.equals(me.regionServer)) {
3638         return false;
3639       }
3640       return (modTime == me.modTime);
3641     }
3642 
3643     @Override
3644     public int hashCode() {
3645       int hash = Arrays.hashCode(getRegionName());
3646       hash ^= getRegionId();
3647       hash ^= Arrays.hashCode(getStartKey());
3648       hash ^= Arrays.hashCode(getEndKey());
3649       hash ^= Boolean.valueOf(isOffline()).hashCode();
3650       hash ^= getTable().hashCode();
3651       if (regionServer != null) {
3652         hash ^= regionServer.hashCode();
3653       }
3654       hash ^= modTime;
3655       return hash;
3656     }
3657   }
3658 
3659   /**
3660    * Stores the regioninfo entries from HDFS
3661    */
3662   static class HdfsEntry {
3663     HRegionInfo hri;
3664     Path hdfsRegionDir = null;
3665     long hdfsRegionDirModTime  = 0;
3666     boolean hdfsRegioninfoFilePresent = false;
3667     boolean hdfsOnlyEdits = false;
3668   }
3669 
3670   /**
3671    * Stores the regioninfo retrieved from Online region servers.
3672    */
3673   static class OnlineEntry {
3674     HRegionInfo hri;
3675     ServerName hsa;
3676 
3677     @Override
3678     public String toString() {
3679       return hsa.toString() + ";" + hri.getRegionNameAsString();
3680     }
3681   }
3682 
3683   /**
3684    * Maintain information about a particular region.  It gathers information
3685    * from three places -- HDFS, META, and region servers.
3686    */
3687   public static class HbckInfo implements KeyRange {
3688     private MetaEntry metaEntry = null; // info in META
3689     private HdfsEntry hdfsEntry = null; // info in HDFS
3690     private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
3691     private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
3692     private boolean skipChecks = false; // whether to skip further checks to this region info.
3693     private boolean isMerged = false;// whether this region has already been merged into another one
3694     private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3695     private HRegionInfo primaryHRIForDeployedReplica = null;
3696 
3697     HbckInfo(MetaEntry metaEntry) {
3698       this.metaEntry = metaEntry;
3699     }
3700 
3701     public synchronized int getReplicaId() {
3702       return metaEntry != null? metaEntry.getReplicaId(): deployedReplicaId;
3703     }
3704 
3705     public synchronized void addServer(HRegionInfo hri, ServerName server) {
3706       OnlineEntry rse = new OnlineEntry() ;
3707       rse.hri = hri;
3708       rse.hsa = server;
3709       this.deployedEntries.add(rse);
3710       this.deployedOn.add(server);
3711       // save the replicaId that we see deployed in the cluster
3712       this.deployedReplicaId = hri.getReplicaId();
3713       this.primaryHRIForDeployedReplica =
3714           RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3715     }
3716 
3717     @Override
3718     public synchronized String toString() {
3719       StringBuilder sb = new StringBuilder();
3720       sb.append("{ meta => ");
3721       sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3722       sb.append( ", hdfs => " + getHdfsRegionDir());
3723       sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3724       sb.append( ", replicaId => " + getReplicaId());
3725       sb.append(" }");
3726       return sb.toString();
3727     }
3728 
3729     @Override
3730     public byte[] getStartKey() {
3731       if (this.metaEntry != null) {
3732         return this.metaEntry.getStartKey();
3733       } else if (this.hdfsEntry != null) {
3734         return this.hdfsEntry.hri.getStartKey();
3735       } else {
3736         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3737         return null;
3738       }
3739     }
3740 
3741     @Override
3742     public byte[] getEndKey() {
3743       if (this.metaEntry != null) {
3744         return this.metaEntry.getEndKey();
3745       } else if (this.hdfsEntry != null) {
3746         return this.hdfsEntry.hri.getEndKey();
3747       } else {
3748         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3749         return null;
3750       }
3751     }
3752 
3753     public TableName getTableName() {
3754       if (this.metaEntry != null) {
3755         return this.metaEntry.getTable();
3756       } else if (this.hdfsEntry != null) {
3757         // we are only guaranteed to have a path and not an HRI for hdfsEntry,
3758         // so we get the name from the Path
3759         Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3760         return FSUtils.getTableName(tableDir);
3761       } else {
3762         // return the info from the first online/deployed hri
3763         for (OnlineEntry e : deployedEntries) {
3764           return e.hri.getTable();
3765         }
3766         return null;
3767       }
3768     }
3769 
3770     public String getRegionNameAsString() {
3771       if (metaEntry != null) {
3772         return metaEntry.getRegionNameAsString();
3773       } else if (hdfsEntry != null) {
3774         if (hdfsEntry.hri != null) {
3775           return hdfsEntry.hri.getRegionNameAsString();
3776         }
3777       } else {
3778         // return the info from the first online/deployed hri
3779         for (OnlineEntry e : deployedEntries) {
3780           return e.hri.getRegionNameAsString();
3781         }
3782       }
3783       return null;
3784     }
3785 
3786     public byte[] getRegionName() {
3787       if (metaEntry != null) {
3788         return metaEntry.getRegionName();
3789       } else if (hdfsEntry != null) {
3790         return hdfsEntry.hri.getRegionName();
3791       } else {
3792         // return the info from the first online/deployed hri
3793         for (OnlineEntry e : deployedEntries) {
3794           return e.hri.getRegionName();
3795         }
3796         return null;
3797       }
3798     }
3799 
3800     public HRegionInfo getPrimaryHRIForDeployedReplica() {
3801       return primaryHRIForDeployedReplica;
3802     }
3803 
3804     Path getHdfsRegionDir() {
3805       if (hdfsEntry == null) {
3806         return null;
3807       }
3808       return hdfsEntry.hdfsRegionDir;
3809     }
3810 
3811     boolean containsOnlyHdfsEdits() {
3812       if (hdfsEntry == null) {
3813         return false;
3814       }
3815       return hdfsEntry.hdfsOnlyEdits;
3816     }
3817 
3818     boolean isHdfsRegioninfoPresent() {
3819       if (hdfsEntry == null) {
3820         return false;
3821       }
3822       return hdfsEntry.hdfsRegioninfoFilePresent;
3823     }
3824 
3825     long getModTime() {
3826       if (hdfsEntry == null) {
3827         return 0;
3828       }
3829       return hdfsEntry.hdfsRegionDirModTime;
3830     }
3831 
3832     HRegionInfo getHdfsHRI() {
3833       if (hdfsEntry == null) {
3834         return null;
3835       }
3836       return hdfsEntry.hri;
3837     }
3838 
3839     public void setSkipChecks(boolean skipChecks) {
3840       this.skipChecks = skipChecks;
3841     }
3842 
3843     public boolean isSkipChecks() {
3844       return skipChecks;
3845     }
3846 
3847     public void setMerged(boolean isMerged) {
3848       this.isMerged = isMerged;
3849     }
3850 
3851     public boolean isMerged() {
3852       return this.isMerged;
3853     }
3854   }
3855 
3856   final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3857     @Override
3858     public int compare(HbckInfo l, HbckInfo r) {
3859       if (l == r) {
3860         // same instance
3861         return 0;
3862       }
3863 
3864       int tableCompare = l.getTableName().compareTo(r.getTableName());
3865       if (tableCompare != 0) {
3866         return tableCompare;
3867       }
3868 
3869       int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3870           l.getStartKey(), r.getStartKey());
3871       if (startComparison != 0) {
3872         return startComparison;
3873       }
3874 
3875       // Special case for absolute endkey
3876       byte[] endKey = r.getEndKey();
3877       endKey = (endKey.length == 0) ? null : endKey;
3878       byte[] endKey2 = l.getEndKey();
3879       endKey2 = (endKey2.length == 0) ? null : endKey2;
3880       int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3881           endKey2,  endKey);
3882 
3883       if (endComparison != 0) {
3884         return endComparison;
3885       }
3886 
3887       // use regionId as tiebreaker.
3888       // Null is considered after all possible values so make it bigger.
3889       if (l.hdfsEntry == null && r.hdfsEntry == null) {
3890         return 0;
3891       }
3892       if (l.hdfsEntry == null && r.hdfsEntry != null) {
3893         return 1;
3894       }
3895       // l.hdfsEntry must not be null
3896       if (r.hdfsEntry == null) {
3897         return -1;
3898       }
3899       // both l.hdfsEntry and r.hdfsEntry must not be null.
3900       return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3901     }
3902   };
3903 
3904   /**
3905    * Prints summary of all tables found on the system.
3906    */
3907   private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3908     StringBuilder sb = new StringBuilder();
3909     int numOfSkippedRegions;
3910     errors.print("Summary:");
3911     for (TableInfo tInfo : tablesInfo.values()) {
3912       numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName())) ?
3913           skippedRegions.get(tInfo.getName()).size() : 0;
3914 
3915       if (errors.tableHasErrors(tInfo)) {
3916         errors.print("Table " + tInfo.getName() + " is inconsistent.");
3917       } else if (numOfSkippedRegions > 0){
3918         errors.print("Table " + tInfo.getName() + " is okay (with "
3919           + numOfSkippedRegions + " skipped regions).");
3920       }
3921       else {
3922         errors.print("Table " + tInfo.getName() + " is okay.");
3923       }
3924       errors.print("    Number of regions: " + tInfo.getNumRegions());
3925       if (numOfSkippedRegions > 0) {
3926         Set<String> skippedRegionStrings = skippedRegions.get(tInfo.getName());
3927         System.out.println("    Number of skipped regions: " + numOfSkippedRegions);
3928         System.out.println("      List of skipped regions:");
3929         for(String sr : skippedRegionStrings) {
3930           System.out.println("        " + sr);
3931         }
3932       }
3933       sb.setLength(0); // clear out existing buffer, if any.
3934       sb.append("    Deployed on: ");
3935       for (ServerName server : tInfo.deployedOn) {
3936         sb.append(" " + server.toString());
3937       }
3938       errors.print(sb.toString());
3939     }
3940   }
3941 
3942   static ErrorReporter getErrorReporter(
3943       final Configuration conf) throws ClassNotFoundException {
3944     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3945     return ReflectionUtils.newInstance(reporter, conf);
3946   }
3947 
3948   public interface ErrorReporter {
3949     enum ERROR_CODE {
3950       UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3951       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
3952       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3953       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3954       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3955       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3956       WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, ORPHANED_ZK_TABLE_ENTRY, BOUNDARIES_ERROR,
3957       DUPE_ENDKEYS
3958     }
3959     void clear();
3960     void report(String message);
3961     void reportError(String message);
3962     void reportError(ERROR_CODE errorCode, String message);
3963     void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3964     void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3965     void reportError(
3966       ERROR_CODE errorCode,
3967       String message,
3968       TableInfo table,
3969       HbckInfo info1,
3970       HbckInfo info2
3971     );
3972     int summarize();
3973     void detail(String details);
3974     ArrayList<ERROR_CODE> getErrorList();
3975     void progress();
3976     void print(String message);
3977     void resetErrors();
3978     boolean tableHasErrors(TableInfo table);
3979   }
3980 
3981   static class PrintingErrorReporter implements ErrorReporter {
3982     public int errorCount = 0;
3983     private int showProgress;
3984     // How frequently calls to progress() will create output
3985     private static final int progressThreshold = 100;
3986 
3987     Set<TableInfo> errorTables = new HashSet<TableInfo>();
3988 
3989     // for use by unit tests to verify which errors were discovered
3990     private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3991 
3992     @Override
3993     public void clear() {
3994       errorTables.clear();
3995       errorList.clear();
3996       errorCount = 0;
3997     }
3998 
3999     @Override
4000     public synchronized void reportError(ERROR_CODE errorCode, String message) {
4001       if (errorCode == ERROR_CODE.WRONG_USAGE) {
4002         System.err.println(message);
4003         return;
4004       }
4005 
4006       errorList.add(errorCode);
4007       if (!getSUMMARY()) {
4008         System.out.println("ERROR: " + message);
4009       }
4010       errorCount++;
4011       showProgress = 0;
4012     }
4013 
4014     @Override
4015     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
4016       errorTables.add(table);
4017       reportError(errorCode, message);
4018     }
4019 
4020     @Override
4021     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
4022                                          HbckInfo info) {
4023       errorTables.add(table);
4024       String reference = "(region " + info.getRegionNameAsString() + ")";
4025       reportError(errorCode, reference + " " + message);
4026     }
4027 
4028     @Override
4029     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
4030                                          HbckInfo info1, HbckInfo info2) {
4031       errorTables.add(table);
4032       String reference = "(regions " + info1.getRegionNameAsString()
4033           + " and " + info2.getRegionNameAsString() + ")";
4034       reportError(errorCode, reference + " " + message);
4035     }
4036 
4037     @Override
4038     public synchronized void reportError(String message) {
4039       reportError(ERROR_CODE.UNKNOWN, message);
4040     }
4041 
4042     /**
4043      * Report error information, but do not increment the error count.  Intended for cases
4044      * where the actual error would have been reported previously.
4045      * @param message
4046      */
4047     @Override
4048     public synchronized void report(String message) {
4049       if (!getSUMMARY()) {
4050         System.out.println("ERROR: " + message);
4051       }
4052       showProgress = 0;
4053     }
4054 
4055     @Override
4056     public synchronized int summarize() {
4057       System.out.println(Integer.toString(errorCount) +
4058                          " inconsistencies detected.");
4059       if (errorCount == 0) {
4060         System.out.println("Status: OK");
4061         return 0;
4062       } else {
4063         System.out.println("Status: INCONSISTENT");
4064         return -1;
4065       }
4066     }
4067 
4068     @Override
4069     public ArrayList<ERROR_CODE> getErrorList() {
4070       return errorList;
4071     }
4072 
4073     @Override
4074     public synchronized void print(String message) {
4075       if (!getSUMMARY()) {
4076         System.out.println(message);
4077       }
4078     }
4079 
4080     private synchronized static boolean getSUMMARY() {
4081       return SUMMARY;
4082     }
4083 
4084     @Override
4085     public boolean tableHasErrors(TableInfo table) {
4086       return errorTables.contains(table);
4087     }
4088 
4089     @Override
4090     public void resetErrors() {
4091       errorCount = 0;
4092     }
4093 
4094     @Override
4095     public synchronized void detail(String message) {
4096       if (details) {
4097         System.out.println(message);
4098       }
4099       showProgress = 0;
4100     }
4101 
4102     @Override
4103     public synchronized void progress() {
4104       if (showProgress++ == progressThreshold) {
4105         if (!getSUMMARY()) {
4106           System.out.print(".");
4107         }
4108         showProgress = 0;
4109       }
4110     }
4111   }
4112 
4113   /**
4114    * Contact a region server and get all information from it
4115    */
4116   static class WorkItemRegion implements Callable<Void> {
4117     private HBaseFsck hbck;
4118     private ServerName rsinfo;
4119     private ErrorReporter errors;
4120     private HConnection connection;
4121 
4122     WorkItemRegion(HBaseFsck hbck, ServerName info,
4123                    ErrorReporter errors, HConnection connection) {
4124       this.hbck = hbck;
4125       this.rsinfo = info;
4126       this.errors = errors;
4127       this.connection = connection;
4128     }
4129 
4130     @Override
4131     public synchronized Void call() throws IOException {
4132       errors.progress();
4133       try {
4134         BlockingInterface server = connection.getAdmin(rsinfo);
4135 
4136         // list all online regions from this region server
4137         List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
4138         regions = filterRegions(regions);
4139 
4140         if (details) {
4141           errors.detail("RegionServer: " + rsinfo.getServerName() +
4142                            " number of regions: " + regions.size());
4143           for (HRegionInfo rinfo: regions) {
4144             errors.detail("  " + rinfo.getRegionNameAsString() +
4145                              " id: " + rinfo.getRegionId() +
4146                              " encoded_name: " + rinfo.getEncodedName() +
4147                              " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
4148                              " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
4149           }
4150         }
4151 
4152         // check to see if the existence of this region matches the region in META
4153         for (HRegionInfo r:regions) {
4154           HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
4155           hbi.addServer(r, rsinfo);
4156         }
4157       } catch (IOException e) {          // unable to connect to the region server.
4158         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
4159           " Unable to fetch region information. " + e);
4160         throw e;
4161       }
4162       return null;
4163     }
4164 
4165     private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
4166       List<HRegionInfo> ret = Lists.newArrayList();
4167       for (HRegionInfo hri : regions) {
4168         if (hri.isMetaTable() || (!hbck.checkMetaOnly
4169             && hbck.isTableIncluded(hri.getTable()))) {
4170           ret.add(hri);
4171         }
4172       }
4173       return ret;
4174     }
4175   }
4176 
4177   /**
4178    * Contact hdfs and get all information about specified table directory into
4179    * regioninfo list.
4180    */
4181   static class WorkItemHdfsDir implements Callable<Void> {
4182     private HBaseFsck hbck;
4183     private FileStatus tableDir;
4184     private ErrorReporter errors;
4185     private FileSystem fs;
4186 
4187     WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
4188                     FileStatus status) {
4189       this.hbck = hbck;
4190       this.fs = fs;
4191       this.tableDir = status;
4192       this.errors = errors;
4193     }
4194 
4195     @Override
4196     public synchronized Void call() throws IOException {
4197       try {
4198         // level 2: <HBASE_DIR>/<table>/*
4199         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
4200         for (FileStatus regionDir : regionDirs) {
4201           errors.progress();
4202           String encodedName = regionDir.getPath().getName();
4203           // ignore directories that aren't hexadecimal
4204           if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
4205             continue;
4206           }
4207 
4208           LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
4209           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
4210           HdfsEntry he = new HdfsEntry();
4211           synchronized (hbi) {
4212             if (hbi.getHdfsRegionDir() != null) {
4213               errors.print("Directory " + encodedName + " duplicate??" +
4214                            hbi.getHdfsRegionDir());
4215             }
4216 
4217             he.hdfsRegionDir = regionDir.getPath();
4218             he.hdfsRegionDirModTime = regionDir.getModificationTime();
4219             Path regioninfoFile = new Path(he.hdfsRegionDir, HRegionFileSystem.REGION_INFO_FILE);
4220             he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
4221             // we add to orphan list when we attempt to read .regioninfo
4222 
4223             // Set a flag if this region contains only edits
4224             // This is special case if a region is left after split
4225             he.hdfsOnlyEdits = true;
4226             FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
4227             Path ePath = WALSplitter.getRegionDirRecoveredEditsDir(regionDir.getPath());
4228             for (FileStatus subDir : subDirs) {
4229               errors.progress();
4230               String sdName = subDir.getPath().getName();
4231               if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
4232                 he.hdfsOnlyEdits = false;
4233                 break;
4234               }
4235             }
4236             hbi.hdfsEntry = he;
4237           }
4238         }
4239       } catch (IOException e) {
4240         // unable to connect to the region server.
4241         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
4242             + tableDir.getPath().getName()
4243             + " Unable to fetch region information. " + e);
4244         throw e;
4245       }
4246       return null;
4247     }
4248   }
4249 
4250   /**
4251    * Contact hdfs and get all information about specified table directory into
4252    * regioninfo list.
4253    */
4254   static class WorkItemHdfsRegionInfo implements Callable<Void> {
4255     private HbckInfo hbi;
4256     private HBaseFsck hbck;
4257     private ErrorReporter errors;
4258 
4259     WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
4260       this.hbi = hbi;
4261       this.hbck = hbck;
4262       this.errors = errors;
4263     }
4264 
4265     @Override
4266     public synchronized Void call() throws IOException {
4267       // only load entries that haven't been loaded yet.
4268       if (hbi.getHdfsHRI() == null) {
4269         try {
4270           errors.progress();
4271           hbck.loadHdfsRegioninfo(hbi);
4272         } catch (IOException ioe) {
4273           String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
4274               + hbi.getTableName() + " in hdfs dir "
4275               + hbi.getHdfsRegionDir()
4276               + "!  It may be an invalid format or version file.  Treating as "
4277               + "an orphaned regiondir.";
4278           errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
4279           try {
4280             hbck.debugLsr(hbi.getHdfsRegionDir());
4281           } catch (IOException ioe2) {
4282             LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
4283             throw ioe2;
4284           }
4285           hbck.orphanHdfsDirs.add(hbi);
4286           throw ioe;
4287         }
4288       }
4289       return null;
4290     }
4291   };
4292 
4293   /**
4294    * Display the full report from fsck. This displays all live and dead region
4295    * servers, and all known regions.
4296    */
4297   public static void setDisplayFullReport() {
4298     details = true;
4299   }
4300 
4301   /**
4302    * Set exclusive mode.
4303    */
4304   public static void setForceExclusive() {
4305     forceExclusive = true;
4306   }
4307 
4308   /**
4309    * Only one instance of hbck can modify HBase at a time.
4310    */
4311   public boolean isExclusive() {
4312     return fixAny || forceExclusive;
4313   }
4314 
4315   /**
4316    * Disable the load balancer.
4317    */
4318   public static void setDisableBalancer() {
4319     disableBalancer = true;
4320   }
4321 
4322   /**
4323    * The balancer should be disabled if we are modifying HBase.
4324    * It can be disabled if you want to prevent region movement from causing
4325    * false positives.
4326    */
4327   public boolean shouldDisableBalancer() {
4328     return fixAny || disableBalancer;
4329   }
4330 
4331   /**
4332    * Set summary mode.
4333    * Print only summary of the tables and status (OK or INCONSISTENT)
4334    */
4335   synchronized static void setSummary() {
4336     SUMMARY = true;
4337   }
4338 
4339   /**
4340    * Set hbase:meta check mode.
4341    * Print only info about hbase:meta table deployment/state
4342    */
4343   void setCheckMetaOnly() {
4344     checkMetaOnly = true;
4345   }
4346 
4347   /**
4348    * Set region boundaries check mode.
4349    */
4350   void setRegionBoundariesCheck() {
4351     checkRegionBoundaries = true;
4352   }
4353 
4354   /**
4355    * Set table locks fix mode.
4356    * Delete table locks held for a long time
4357    */
4358   public void setFixTableLocks(boolean shouldFix) {
4359     fixTableLocks = shouldFix;
4360     fixAny |= shouldFix;
4361   }
4362 
4363   /**
4364    * Set orphaned table ZNodes fix mode.
4365    * Set the table state to disable in the orphaned table ZNode.
4366    */
4367   public void setFixTableZNodes(boolean shouldFix) {
4368     fixTableZNodes = shouldFix;
4369     fixAny |= shouldFix;
4370   }
4371 
4372   /**
4373    * Check if we should rerun fsck again. This checks if we've tried to
4374    * fix something and we should rerun fsck tool again.
4375    * Display the full report from fsck. This displays all live and dead
4376    * region servers, and all known regions.
4377    */
4378   void setShouldRerun() {
4379     rerun = true;
4380   }
4381 
4382   boolean shouldRerun() {
4383     return rerun;
4384   }
4385 
4386   /**
4387    * Fix inconsistencies found by fsck. This should try to fix errors (if any)
4388    * found by fsck utility.
4389    */
4390   public void setFixAssignments(boolean shouldFix) {
4391     fixAssignments = shouldFix;
4392     fixAny |= shouldFix;
4393   }
4394 
4395   boolean shouldFixAssignments() {
4396     return fixAssignments;
4397   }
4398 
4399   public void setFixMeta(boolean shouldFix) {
4400     fixMeta = shouldFix;
4401     fixAny |= shouldFix;
4402   }
4403 
4404   boolean shouldFixMeta() {
4405     return fixMeta;
4406   }
4407 
4408   public void setFixEmptyMetaCells(boolean shouldFix) {
4409     fixEmptyMetaCells = shouldFix;
4410     fixAny |= shouldFix;
4411   }
4412 
4413   boolean shouldFixEmptyMetaCells() {
4414     return fixEmptyMetaCells;
4415   }
4416 
4417   public void setCheckHdfs(boolean checking) {
4418     checkHdfs = checking;
4419   }
4420 
4421   boolean shouldCheckHdfs() {
4422     return checkHdfs;
4423   }
4424 
4425   public void setFixHdfsHoles(boolean shouldFix) {
4426     fixHdfsHoles = shouldFix;
4427     fixAny |= shouldFix;
4428   }
4429 
4430   boolean shouldFixHdfsHoles() {
4431     return fixHdfsHoles;
4432   }
4433 
4434   public void setFixTableOrphans(boolean shouldFix) {
4435     fixTableOrphans = shouldFix;
4436     fixAny |= shouldFix;
4437   }
4438 
4439   boolean shouldFixTableOrphans() {
4440     return fixTableOrphans;
4441   }
4442 
4443   public void setFixHdfsOverlaps(boolean shouldFix) {
4444     fixHdfsOverlaps = shouldFix;
4445     fixAny |= shouldFix;
4446   }
4447 
4448   boolean shouldFixHdfsOverlaps() {
4449     return fixHdfsOverlaps;
4450   }
4451 
4452   public void setFixHdfsOrphans(boolean shouldFix) {
4453     fixHdfsOrphans = shouldFix;
4454     fixAny |= shouldFix;
4455   }
4456 
4457   boolean shouldFixHdfsOrphans() {
4458     return fixHdfsOrphans;
4459   }
4460 
4461   public void setFixVersionFile(boolean shouldFix) {
4462     fixVersionFile = shouldFix;
4463     fixAny |= shouldFix;
4464   }
4465 
4466   public boolean shouldFixVersionFile() {
4467     return fixVersionFile;
4468   }
4469 
4470   public void setSidelineBigOverlaps(boolean sbo) {
4471     this.sidelineBigOverlaps = sbo;
4472   }
4473 
4474   public boolean shouldSidelineBigOverlaps() {
4475     return sidelineBigOverlaps;
4476   }
4477 
4478   public void setFixSplitParents(boolean shouldFix) {
4479     fixSplitParents = shouldFix;
4480     fixAny |= shouldFix;
4481   }
4482 
4483   public void setRemoveParents(boolean shouldFix) {
4484     removeParents = shouldFix;
4485     fixAny |= shouldFix;
4486   }
4487 
4488   boolean shouldFixSplitParents() {
4489     return fixSplitParents;
4490   }
4491 
4492   boolean shouldRemoveParents() {
4493     return removeParents;
4494   }
4495 
4496   public void setFixReferenceFiles(boolean shouldFix) {
4497     fixReferenceFiles = shouldFix;
4498     fixAny |= shouldFix;
4499   }
4500 
4501   boolean shouldFixReferenceFiles() {
4502     return fixReferenceFiles;
4503   }
4504 
4505   public boolean shouldIgnorePreCheckPermission() {
4506     return !fixAny || ignorePreCheckPermission;
4507   }
4508 
4509   public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
4510     this.ignorePreCheckPermission = ignorePreCheckPermission;
4511   }
4512 
4513   /**
4514    * @param mm maximum number of regions to merge into a single region.
4515    */
4516   public void setMaxMerge(int mm) {
4517     this.maxMerge = mm;
4518   }
4519 
4520   public int getMaxMerge() {
4521     return maxMerge;
4522   }
4523 
4524   public void setMaxOverlapsToSideline(int mo) {
4525     this.maxOverlapsToSideline = mo;
4526   }
4527 
4528   public int getMaxOverlapsToSideline() {
4529     return maxOverlapsToSideline;
4530   }
4531 
4532   /**
4533    * Only check/fix tables specified by the list,
4534    * Empty list means all tables are included.
4535    */
4536   boolean isTableIncluded(TableName table) {
4537     return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
4538   }
4539 
4540   public void includeTable(TableName table) {
4541     tablesIncluded.add(table);
4542   }
4543 
4544   Set<TableName> getIncludedTables() {
4545     return new HashSet<TableName>(tablesIncluded);
4546   }
4547 
4548   /**
4549    * We are interested in only those tables that have not changed their state in
4550    * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
4551    * @param seconds - the time in seconds
4552    */
4553   public void setTimeLag(long seconds) {
4554     timelag = seconds * 1000; // convert to milliseconds
4555   }
4556 
4557   /**
4558    *
4559    * @param sidelineDir - HDFS path to sideline data
4560    */
4561   public void setSidelineDir(String sidelineDir) {
4562     this.sidelineDir = new Path(sidelineDir);
4563   }
4564 
4565   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
4566     return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
4567   }
4568 
4569   public HFileCorruptionChecker getHFilecorruptionChecker() {
4570     return hfcc;
4571   }
4572 
4573   public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
4574     this.hfcc = hfcc;
4575   }
4576 
4577   public void setRetCode(int code) {
4578     this.retcode = code;
4579   }
4580 
4581   public int getRetCode() {
4582     return retcode;
4583   }
4584 
4585   protected HBaseFsck printUsageAndExit() {
4586     StringWriter sw = new StringWriter(2048);
4587     PrintWriter out = new PrintWriter(sw);
4588     out.println("Usage: fsck [opts] {only tables}");
4589     out.println(" where [opts] are:");
4590     out.println("   -help Display help options (this)");
4591     out.println("   -details Display full report of all regions.");
4592     out.println("   -timelag <timeInSeconds>  Process only regions that " +
4593                        " have not experienced any metadata updates in the last " +
4594                        " <timeInSeconds> seconds.");
4595     out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
4596         " before checking if the fix worked if run with -fix");
4597     out.println("   -summary Print only summary of the tables and status.");
4598     out.println("   -metaonly Only check the state of the hbase:meta table.");
4599     out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
4600     out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
4601     out.println("   -exclusive Abort if another hbck is exclusive or fixing.");
4602     out.println("   -disableBalancer Disable the load balancer.");
4603 
4604     out.println("");
4605     out.println("  Metadata Repair options: (expert features, use with caution!)");
4606     out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
4607     out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
4608     out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
4609     out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
4610         + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
4611     out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
4612     out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
4613     out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
4614     out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
4615     out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
4616     out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
4617     out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
4618     out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
4619     out.println("   -fixSplitParents  Try to force offline split parents to be online.");
4620     out.println("   -removeParents    Try to offline and sideline lingering parents and keep daughter regions.");
4621     out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
4622     out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
4623     out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
4624         + " (empty REGIONINFO_QUALIFIER rows)");
4625 
4626     out.println("");
4627     out.println("  Datafile Repair options: (expert features, use with caution!)");
4628     out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
4629     out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
4630 
4631     out.println("");
4632     out.println("  Metadata Repair shortcuts");
4633     out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
4634         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps " +
4635         "-fixReferenceFiles -fixTableLocks -fixOrphanedTableZnodes");
4636     out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
4637 
4638     out.println("");
4639     out.println("  Table lock options");
4640     out.println("   -fixTableLocks    Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
4641 
4642     out.println("");
4643     out.println("  Table Znode options");
4644     out.println("   -fixOrphanedTableZnodes    Set table state in ZNode to disabled if table does not exists");
4645 
4646     out.flush();
4647     errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
4648 
4649     setRetCode(-2);
4650     return this;
4651   }
4652 
4653   /**
4654    * Main program
4655    *
4656    * @param args
4657    * @throws Exception
4658    */
4659   public static void main(String[] args) throws Exception {
4660     // create a fsck object
4661     Configuration conf = HBaseConfiguration.create();
4662     Path hbasedir = FSUtils.getRootDir(conf);
4663     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
4664     FSUtils.setFsDefault(conf, new Path(defaultFs));
4665     int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
4666     System.exit(ret);
4667   }
4668 
4669   /**
4670    * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
4671    */
4672   static class HBaseFsckTool extends Configured implements Tool {
4673     HBaseFsckTool(Configuration conf) { super(conf); }
4674     @Override
4675     public int run(String[] args) throws Exception {
4676       HBaseFsck hbck = new HBaseFsck(getConf());
4677       hbck.exec(hbck.executor, args);
4678       hbck.close();
4679       return hbck.getRetCode();
4680     }
4681   };
4682 
4683 
4684   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
4685     ServiceException, InterruptedException {
4686     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
4687 
4688     boolean checkCorruptHFiles = false;
4689     boolean sidelineCorruptHFiles = false;
4690 
4691     // Process command-line args.
4692     for (int i = 0; i < args.length; i++) {
4693       String cmd = args[i];
4694       if (cmd.equals("-help") || cmd.equals("-h")) {
4695         return printUsageAndExit();
4696       } else if (cmd.equals("-details")) {
4697         setDisplayFullReport();
4698       } else if (cmd.equals("-exclusive")) {
4699         setForceExclusive();
4700       } else if (cmd.equals("-disableBalancer")) {
4701         setDisableBalancer();
4702       } else if (cmd.equals("-timelag")) {
4703         if (i == args.length - 1) {
4704           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4705           return printUsageAndExit();
4706         }
4707         try {
4708           long timelag = Long.parseLong(args[i+1]);
4709           setTimeLag(timelag);
4710         } catch (NumberFormatException e) {
4711           errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4712           return printUsageAndExit();
4713         }
4714         i++;
4715       } else if (cmd.equals("-sleepBeforeRerun")) {
4716         if (i == args.length - 1) {
4717           errors.reportError(ERROR_CODE.WRONG_USAGE,
4718             "HBaseFsck: -sleepBeforeRerun needs a value.");
4719           return printUsageAndExit();
4720         }
4721         try {
4722           sleepBeforeRerun = Long.parseLong(args[i+1]);
4723         } catch (NumberFormatException e) {
4724           errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4725           return printUsageAndExit();
4726         }
4727         i++;
4728       } else if (cmd.equals("-sidelineDir")) {
4729         if (i == args.length - 1) {
4730           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4731           return printUsageAndExit();
4732         }
4733         i++;
4734         setSidelineDir(args[i]);
4735       } else if (cmd.equals("-fix")) {
4736         errors.reportError(ERROR_CODE.WRONG_USAGE,
4737           "This option is deprecated, please use  -fixAssignments instead.");
4738         setFixAssignments(true);
4739       } else if (cmd.equals("-fixAssignments")) {
4740         setFixAssignments(true);
4741       } else if (cmd.equals("-fixMeta")) {
4742         setFixMeta(true);
4743       } else if (cmd.equals("-noHdfsChecking")) {
4744         setCheckHdfs(false);
4745       } else if (cmd.equals("-fixHdfsHoles")) {
4746         setFixHdfsHoles(true);
4747       } else if (cmd.equals("-fixHdfsOrphans")) {
4748         setFixHdfsOrphans(true);
4749       } else if (cmd.equals("-fixTableOrphans")) {
4750         setFixTableOrphans(true);
4751       } else if (cmd.equals("-fixHdfsOverlaps")) {
4752         setFixHdfsOverlaps(true);
4753       } else if (cmd.equals("-fixVersionFile")) {
4754         setFixVersionFile(true);
4755       } else if (cmd.equals("-sidelineBigOverlaps")) {
4756         setSidelineBigOverlaps(true);
4757       } else if (cmd.equals("-fixSplitParents")) {
4758         setFixSplitParents(true);
4759       } else if (cmd.equals("-removeParents")) {
4760         setRemoveParents(true);
4761       } else if (cmd.equals("-ignorePreCheckPermission")) {
4762         setIgnorePreCheckPermission(true);
4763       } else if (cmd.equals("-checkCorruptHFiles")) {
4764         checkCorruptHFiles = true;
4765       } else if (cmd.equals("-sidelineCorruptHFiles")) {
4766         sidelineCorruptHFiles = true;
4767       } else if (cmd.equals("-fixReferenceFiles")) {
4768         setFixReferenceFiles(true);
4769       } else if (cmd.equals("-fixEmptyMetaCells")) {
4770         setFixEmptyMetaCells(true);
4771       } else if (cmd.equals("-repair")) {
4772         // this attempts to merge overlapping hdfs regions, needs testing
4773         // under load
4774         setFixHdfsHoles(true);
4775         setFixHdfsOrphans(true);
4776         setFixMeta(true);
4777         setFixAssignments(true);
4778         setFixHdfsOverlaps(true);
4779         setFixVersionFile(true);
4780         setSidelineBigOverlaps(true);
4781         setFixSplitParents(false);
4782         setCheckHdfs(true);
4783         setFixReferenceFiles(true);
4784         setFixTableLocks(true);
4785         setFixTableZNodes(true);
4786       } else if (cmd.equals("-repairHoles")) {
4787         // this will make all missing hdfs regions available but may lose data
4788         setFixHdfsHoles(true);
4789         setFixHdfsOrphans(false);
4790         setFixMeta(true);
4791         setFixAssignments(true);
4792         setFixHdfsOverlaps(false);
4793         setSidelineBigOverlaps(false);
4794         setFixSplitParents(false);
4795         setCheckHdfs(true);
4796       } else if (cmd.equals("-maxOverlapsToSideline")) {
4797         if (i == args.length - 1) {
4798           errors.reportError(ERROR_CODE.WRONG_USAGE,
4799             "-maxOverlapsToSideline needs a numeric value argument.");
4800           return printUsageAndExit();
4801         }
4802         try {
4803           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4804           setMaxOverlapsToSideline(maxOverlapsToSideline);
4805         } catch (NumberFormatException e) {
4806           errors.reportError(ERROR_CODE.WRONG_USAGE,
4807             "-maxOverlapsToSideline needs a numeric value argument.");
4808           return printUsageAndExit();
4809         }
4810         i++;
4811       } else if (cmd.equals("-maxMerge")) {
4812         if (i == args.length - 1) {
4813           errors.reportError(ERROR_CODE.WRONG_USAGE,
4814             "-maxMerge needs a numeric value argument.");
4815           return printUsageAndExit();
4816         }
4817         try {
4818           int maxMerge = Integer.parseInt(args[i+1]);
4819           setMaxMerge(maxMerge);
4820         } catch (NumberFormatException e) {
4821           errors.reportError(ERROR_CODE.WRONG_USAGE,
4822             "-maxMerge needs a numeric value argument.");
4823           return printUsageAndExit();
4824         }
4825         i++;
4826       } else if (cmd.equals("-summary")) {
4827         setSummary();
4828       } else if (cmd.equals("-metaonly")) {
4829         setCheckMetaOnly();
4830       } else if (cmd.equals("-boundaries")) {
4831         setRegionBoundariesCheck();
4832       } else if (cmd.equals("-fixTableLocks")) {
4833         setFixTableLocks(true);
4834       } else if (cmd.equals("-fixOrphanedTableZnodes")) {
4835         setFixTableZNodes(true);
4836       } else if (cmd.startsWith("-")) {
4837         errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4838         return printUsageAndExit();
4839       } else {
4840         includeTable(TableName.valueOf(cmd));
4841         errors.print("Allow checking/fixes for table: " + cmd);
4842       }
4843     }
4844 
4845     errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
4846 
4847     // pre-check current user has FS write permission or not
4848     try {
4849       preCheckPermission();
4850     } catch (AccessDeniedException ace) {
4851       Runtime.getRuntime().exit(-1);
4852     } catch (IOException ioe) {
4853       Runtime.getRuntime().exit(-1);
4854     }
4855 
4856     // do the real work of hbck
4857     connect();
4858 
4859     try {
4860       // if corrupt file mode is on, first fix them since they may be opened later
4861       if (checkCorruptHFiles || sidelineCorruptHFiles) {
4862         LOG.info("Checking all hfiles for corruption");
4863         HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4864         setHFileCorruptionChecker(hfcc); // so we can get result
4865         Collection<TableName> tables = getIncludedTables();
4866         Collection<Path> tableDirs = new ArrayList<Path>();
4867         Path rootdir = FSUtils.getRootDir(getConf());
4868         if (tables.size() > 0) {
4869           for (TableName t : tables) {
4870             tableDirs.add(FSUtils.getTableDir(rootdir, t));
4871           }
4872         } else {
4873           tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4874         }
4875         hfcc.checkTables(tableDirs);
4876         hfcc.report(errors);
4877       }
4878 
4879       // check and fix table integrity, region consistency.
4880       int code = onlineHbck();
4881       setRetCode(code);
4882       // If we have changed the HBase state it is better to run hbck again
4883       // to see if we haven't broken something else in the process.
4884       // We run it only once more because otherwise we can easily fall into
4885       // an infinite loop.
4886       if (shouldRerun()) {
4887         try {
4888           LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4889           Thread.sleep(sleepBeforeRerun);
4890         } catch (InterruptedException ie) {
4891           LOG.warn("Interrupted while sleeping");
4892           return this;
4893         }
4894         // Just report
4895         setFixAssignments(false);
4896         setFixMeta(false);
4897         setFixHdfsHoles(false);
4898         setFixHdfsOverlaps(false);
4899         setFixVersionFile(false);
4900         setFixTableOrphans(false);
4901         errors.resetErrors();
4902         code = onlineHbck();
4903         setRetCode(code);
4904       }
4905     } finally {
4906       IOUtils.closeQuietly(this);
4907     }
4908     return this;
4909   }
4910 
4911   /**
4912    * ls -r for debugging purposes
4913    */
4914   void debugLsr(Path p) throws IOException {
4915     debugLsr(getConf(), p, errors);
4916   }
4917 
4918   /**
4919    * ls -r for debugging purposes
4920    */
4921   public static void debugLsr(Configuration conf,
4922       Path p) throws IOException {
4923     debugLsr(conf, p, new PrintingErrorReporter());
4924   }
4925 
4926   /**
4927    * ls -r for debugging purposes
4928    */
4929   public static void debugLsr(Configuration conf,
4930       Path p, ErrorReporter errors) throws IOException {
4931     if (!LOG.isDebugEnabled() || p == null) {
4932       return;
4933     }
4934     FileSystem fs = p.getFileSystem(conf);
4935 
4936     if (!fs.exists(p)) {
4937       // nothing
4938       return;
4939     }
4940     errors.print(p.toString());
4941 
4942     if (fs.isFile(p)) {
4943       return;
4944     }
4945 
4946     if (fs.getFileStatus(p).isDirectory()) {
4947       FileStatus[] fss= fs.listStatus(p);
4948       for (FileStatus status : fss) {
4949         debugLsr(conf, status.getPath(), errors);
4950       }
4951     }
4952   }
4953 }