View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.Closeable;
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.io.PrintWriter;
24  import java.io.StringWriter;
25  import java.net.InetAddress;
26  import java.net.URI;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.Collection;
30  import java.util.Collections;
31  import java.util.Comparator;
32  import java.util.HashMap;
33  import java.util.HashSet;
34  import java.util.Iterator;
35  import java.util.List;
36  import java.util.Map;
37  import java.util.Map.Entry;
38  import java.util.Set;
39  import java.util.SortedMap;
40  import java.util.SortedSet;
41  import java.util.TreeMap;
42  import java.util.TreeSet;
43  import java.util.concurrent.Callable;
44  import java.util.concurrent.ConcurrentSkipListMap;
45  import java.util.concurrent.ExecutionException;
46  import java.util.concurrent.ExecutorService;
47  import java.util.concurrent.Executors;
48  import java.util.concurrent.Future;
49  import java.util.concurrent.FutureTask;
50  import java.util.concurrent.ScheduledThreadPoolExecutor;
51  import java.util.concurrent.TimeUnit;
52  import java.util.concurrent.TimeoutException;
53  import java.util.concurrent.atomic.AtomicBoolean;
54  import java.util.concurrent.atomic.AtomicInteger;
55  
56  import org.apache.commons.lang.RandomStringUtils;
57  import org.apache.commons.lang.StringUtils;
58  import org.apache.commons.logging.Log;
59  import org.apache.commons.logging.LogFactory;
60  import org.apache.hadoop.hbase.classification.InterfaceAudience;
61  import org.apache.hadoop.hbase.classification.InterfaceStability;
62  import org.apache.hadoop.conf.Configuration;
63  import org.apache.hadoop.conf.Configured;
64  import org.apache.hadoop.fs.FSDataOutputStream;
65  import org.apache.hadoop.fs.FileStatus;
66  import org.apache.hadoop.fs.FileSystem;
67  import org.apache.hadoop.fs.Path;
68  import org.apache.hadoop.fs.permission.FsAction;
69  import org.apache.hadoop.fs.permission.FsPermission;
70  import org.apache.hadoop.hbase.Abortable;
71  import org.apache.hadoop.hbase.Cell;
72  import org.apache.hadoop.hbase.ClusterStatus;
73  import org.apache.hadoop.hbase.HBaseConfiguration;
74  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
75  import org.apache.hadoop.hbase.HColumnDescriptor;
76  import org.apache.hadoop.hbase.HConstants;
77  import org.apache.hadoop.hbase.HRegionInfo;
78  import org.apache.hadoop.hbase.HRegionLocation;
79  import org.apache.hadoop.hbase.HTableDescriptor;
80  import org.apache.hadoop.hbase.KeyValue;
81  import org.apache.hadoop.hbase.MasterNotRunningException;
82  import org.apache.hadoop.hbase.MetaTableAccessor;
83  import org.apache.hadoop.hbase.RegionLocations;
84  import org.apache.hadoop.hbase.ServerName;
85  import org.apache.hadoop.hbase.TableDescriptor;
86  import org.apache.hadoop.hbase.TableName;
87  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
88  import org.apache.hadoop.hbase.client.Admin;
89  import org.apache.hadoop.hbase.client.ClusterConnection;
90  import org.apache.hadoop.hbase.client.ConnectionFactory;
91  import org.apache.hadoop.hbase.client.Delete;
92  import org.apache.hadoop.hbase.client.Get;
93  import org.apache.hadoop.hbase.client.HBaseAdmin;
94  import org.apache.hadoop.hbase.client.HConnectable;
95  import org.apache.hadoop.hbase.client.HConnection;
96  import org.apache.hadoop.hbase.client.HConnectionManager;
97  import org.apache.hadoop.hbase.client.MetaScanner;
98  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
99  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
100 import org.apache.hadoop.hbase.client.Put;
101 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
102 import org.apache.hadoop.hbase.client.Result;
103 import org.apache.hadoop.hbase.client.RowMutations;
104 import org.apache.hadoop.hbase.client.Table;
105 import org.apache.hadoop.hbase.client.TableState;
106 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
107 import org.apache.hadoop.hbase.io.hfile.HFile;
108 import org.apache.hadoop.hbase.master.MasterFileSystem;
109 import org.apache.hadoop.hbase.master.RegionState;
110 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
111 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
112 import org.apache.hadoop.hbase.regionserver.HRegion;
113 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
114 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
115 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
116 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
117 import org.apache.hadoop.hbase.security.AccessDeniedException;
118 import org.apache.hadoop.hbase.security.UserProvider;
119 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
120 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
121 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
122 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
123 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
124 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
125 import org.apache.hadoop.hbase.wal.WAL;
126 import org.apache.hadoop.hbase.wal.WALFactory;
127 import org.apache.hadoop.hbase.wal.WALSplitter;
128 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
129 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
130 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
131 import org.apache.hadoop.io.IOUtils;
132 import org.apache.hadoop.ipc.RemoteException;
133 import org.apache.hadoop.security.UserGroupInformation;
134 import org.apache.hadoop.util.ReflectionUtils;
135 import org.apache.hadoop.util.Tool;
136 import org.apache.hadoop.util.ToolRunner;
137 import org.apache.zookeeper.KeeperException;
138 
139 import com.google.common.base.Joiner;
140 import com.google.common.base.Preconditions;
141 import com.google.common.collect.Lists;
142 import com.google.common.collect.Multimap;
143 import com.google.common.collect.TreeMultimap;
144 import com.google.protobuf.ServiceException;
145 
146 /**
147  * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
148  * table integrity problems in a corrupted HBase.
149  * <p>
150  * Region consistency checks verify that hbase:meta, region deployment on region
151  * servers and the state of data in HDFS (.regioninfo files) all are in
152  * accordance.
153  * <p>
154  * Table integrity checks verify that all possible row keys resolve to exactly
155  * one region of a table.  This means there are no individual degenerate
156  * or backwards regions; no holes between regions; and that there are no
157  * overlapping regions.
158  * <p>
159  * The general repair strategy works in two phases:
160  * <ol>
161  * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
162  * <li> Repair Region Consistency with hbase:meta and assignments
163  * </ol>
164  * <p>
165  * For table integrity repairs, the tables' region directories are scanned
166  * for .regioninfo files.  Each table's integrity is then verified.  If there
167  * are any orphan regions (regions with no .regioninfo files) or holes, new
168  * regions are fabricated.  Backwards regions are sidelined as well as empty
169  * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
170  * a new region is created and all data is merged into the new region.
171  * <p>
172  * Table integrity repairs deal solely with HDFS and could potentially be done
173  * offline -- the hbase region servers or master do not need to be running.
174  * This phase can eventually be used to completely reconstruct the hbase:meta table in
175  * an offline fashion.
176  * <p>
177  * Region consistency requires three conditions -- 1) valid .regioninfo file
178  * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
179  * and 3) a region is deployed only at the regionserver that was assigned to
180  * with proper state in the master.
181  * <p>
182  * Region consistency repairs require hbase to be online so that hbck can
183  * contact the HBase master and region servers.  The hbck#connect() method must
184  * first be called successfully.  Much of the region consistency information
185  * is transient and less risky to repair.
186  * <p>
187  * If hbck is run from the command line, there are a handful of arguments that
188  * can be used to limit the kinds of repairs hbck will do.  See the code in
189  * {@link #printUsageAndExit()} for more details.
190  */
191 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
192 @InterfaceStability.Evolving
193 public class HBaseFsck extends Configured implements Closeable {
194   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
195   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
196   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
197   private static boolean rsSupportsOffline = true;
198   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
199   private static final int DEFAULT_MAX_MERGE = 5;
200   private static final String TO_BE_LOADED = "to_be_loaded";
201   private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
202 
203 
204   /**********************
205    * Internal resources
206    **********************/
207   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
208   private ClusterStatus status;
209   private ClusterConnection connection;
210   private Admin admin;
211   private Table meta;
212   // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
213   protected ExecutorService executor;
214   private long startMillis = System.currentTimeMillis();
215   private HFileCorruptionChecker hfcc;
216   private int retcode = 0;
217   private Path HBCK_LOCK_PATH;
218   private FSDataOutputStream hbckOutFd;
219   // This lock is to prevent cleanup of balancer resources twice between
220   // ShutdownHook and the main code. We cleanup only if the connect() is
221   // successful
222   private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
223 
224   /***********
225    * Options
226    ***********/
227   private static boolean details = false; // do we display the full report
228   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
229   private boolean fixAssignments = false; // fix assignment errors?
230   private boolean fixMeta = false; // fix meta errors?
231   private boolean checkHdfs = true; // load and check fs consistency?
232   private boolean fixHdfsHoles = false; // fix fs holes?
233   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
234   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
235   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
236   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
237   private boolean fixSplitParents = false; // fix lingering split parents
238   private boolean fixReferenceFiles = false; // fix lingering reference store file
239   private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
240   private boolean fixTableLocks = false; // fix table locks which are expired
241   private boolean fixAny = false; // Set to true if any of the fix is required.
242 
243   // limit checking/fixes to listed tables, if empty attempt to check/fix all
244   // hbase:meta are always checked
245   private Set<TableName> tablesIncluded = new HashSet<TableName>();
246   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
247   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
248   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
249   private Path sidelineDir = null;
250 
251   private boolean rerun = false; // if we tried to fix something, rerun hbck
252   private static boolean summary = false; // if we want to print less output
253   private boolean checkMetaOnly = false;
254   private boolean checkRegionBoundaries = false;
255   private boolean ignorePreCheckPermission = false; // if pre-check permission
256 
257   /*********
258    * State
259    *********/
260   final private ErrorReporter errors;
261   int fixes = 0;
262 
263   /**
264    * This map contains the state of all hbck items.  It maps from encoded region
265    * name to HbckInfo structure.  The information contained in HbckInfo is used
266    * to detect and correct consistency (hdfs/meta/deployment) problems.
267    */
268   private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
269   private TreeSet<TableName> disabledTables =
270     new TreeSet<TableName>();
271   // Empty regioninfo qualifiers in hbase:meta
272   private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
273 
274   /**
275    * This map from Tablename -> TableInfo contains the structures necessary to
276    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
277    * to prevent dupes.
278    *
279    * If tablesIncluded is empty, this map contains all tables.
280    * Otherwise, it contains only meta tables and tables in tablesIncluded,
281    * unless checkMetaOnly is specified, in which case, it contains only
282    * the meta table
283    */
284   private SortedMap<TableName, TableInfo> tablesInfo =
285       new ConcurrentSkipListMap<TableName, TableInfo>();
286 
287   /**
288    * When initially looking at HDFS, we attempt to find any orphaned data.
289    */
290   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
291 
292   private Map<TableName, Set<String>> orphanTableDirs =
293       new HashMap<TableName, Set<String>>();
294 
295   /**
296    * Constructor
297    *
298    * @param conf Configuration object
299    * @throws MasterNotRunningException if the master is not running
300    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
301    */
302   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
303       ZooKeeperConnectionException, IOException, ClassNotFoundException {
304     super(conf);
305     // make a copy, just to be sure we're not overriding someone else's config
306     setConf(HBaseConfiguration.create(getConf()));
307     // disable blockcache for tool invocation, see HBASE-10500
308     getConf().setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0);
309     errors = getErrorReporter(conf);
310 
311     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
312     executor = new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
313   }
314 
315   /**
316    * Constructor
317    *
318    * @param conf
319    *          Configuration object
320    * @throws MasterNotRunningException
321    *           if the master is not running
322    * @throws ZooKeeperConnectionException
323    *           if unable to connect to ZooKeeper
324    */
325   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
326       ZooKeeperConnectionException, IOException, ClassNotFoundException {
327     super(conf);
328     errors = getErrorReporter(getConf());
329     this.executor = exec;
330   }
331   
332   private class FileLockCallable implements Callable<FSDataOutputStream> {
333     @Override
334     public FSDataOutputStream call() throws IOException {
335       try {
336         FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
337         FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
338             HConstants.DATA_FILE_UMASK_KEY);
339         Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
340         fs.mkdirs(tmpDir);
341         HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
342         final FSDataOutputStream out = FSUtils.create(fs, HBCK_LOCK_PATH, defaultPerms, false);
343         out.writeBytes(InetAddress.getLocalHost().toString());
344         out.flush();
345         return out;
346       } catch(RemoteException e) {
347         if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
348           return null;
349         } else {
350           throw e;
351         }
352       }
353     }
354   }
355 
356   /**
357    * This method maintains a lock using a file. If the creation fails we return null
358    *
359    * @return FSDataOutputStream object corresponding to the newly opened lock file
360    * @throws IOException
361    */
362   private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
363     FileLockCallable callable = new FileLockCallable();
364     ExecutorService executor = Executors.newFixedThreadPool(1);
365     FutureTask<FSDataOutputStream> futureTask = new FutureTask<FSDataOutputStream>(callable);
366     executor.execute(futureTask);
367     final int timeoutInSeconds = 30;
368     FSDataOutputStream stream = null;
369     try {
370       stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);
371     } catch (ExecutionException ee) {
372       LOG.warn("Encountered exception when opening lock file", ee);
373     } catch (InterruptedException ie) {
374       LOG.warn("Interrupted when opening lock file", ie);
375       Thread.currentThread().interrupt();
376     } catch (TimeoutException exception) {
377       // took too long to obtain lock
378       LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");
379       futureTask.cancel(true);
380     } finally {
381       executor.shutdownNow();
382     }
383     return stream;
384   }
385 
386   private void unlockHbck() {
387     if(hbckLockCleanup.compareAndSet(true, false)){
388       IOUtils.closeStream(hbckOutFd);
389       try{
390         FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()), HBCK_LOCK_PATH, true);
391       } catch(IOException ioe) {
392         LOG.warn("Failed to delete " + HBCK_LOCK_PATH);
393         LOG.debug(ioe);
394       }
395     }
396   }
397 
398   /**
399    * To repair region consistency, one must call connect() in order to repair
400    * online state.
401    */
402   public void connect() throws IOException {
403 
404     // Check if another instance of balancer is running
405     hbckOutFd = checkAndMarkRunningHbck();
406     if (hbckOutFd == null) {
407       setRetCode(-1);
408       LOG.error("Another instance of hbck is running, exiting this instance.[If you are sure" +
409           " no other instance is running, delete the lock file " +
410           HBCK_LOCK_PATH + " and rerun the tool]");
411       throw new IOException("Duplicate hbck - Abort");
412     }
413 
414     // Make sure to cleanup the lock
415     hbckLockCleanup.set(true);
416 
417     // Add a shutdown hook to this thread, incase user tries to
418     // kill the hbck with a ctrl-c, we want to cleanup the lock so that
419     // it is available for further calls
420     Runtime.getRuntime().addShutdownHook(new Thread() {
421       @Override
422       public void run() {
423         IOUtils.closeStream(HBaseFsck.this);
424         unlockHbck();
425       }
426     });
427     LOG.debug("Launching hbck");
428 
429     connection = (ClusterConnection)ConnectionFactory.createConnection(getConf());
430     admin = connection.getAdmin();
431     meta = connection.getTable(TableName.META_TABLE_NAME);
432     status = admin.getClusterStatus();
433   }
434 
435   /**
436    * Get deployed regions according to the region servers.
437    */
438   private void loadDeployedRegions() throws IOException, InterruptedException {
439     // From the master, get a list of all known live region servers
440     Collection<ServerName> regionServers = status.getServers();
441     errors.print("Number of live region servers: " + regionServers.size());
442     if (details) {
443       for (ServerName rsinfo: regionServers) {
444         errors.print("  " + rsinfo.getServerName());
445       }
446     }
447 
448     // From the master, get a list of all dead region servers
449     Collection<ServerName> deadRegionServers = status.getDeadServerNames();
450     errors.print("Number of dead region servers: " + deadRegionServers.size());
451     if (details) {
452       for (ServerName name: deadRegionServers) {
453         errors.print("  " + name);
454       }
455     }
456 
457     // Print the current master name and state
458     errors.print("Master: " + status.getMaster());
459 
460     // Print the list of all backup masters
461     Collection<ServerName> backupMasters = status.getBackupMasters();
462     errors.print("Number of backup masters: " + backupMasters.size());
463     if (details) {
464       for (ServerName name: backupMasters) {
465         errors.print("  " + name);
466       }
467     }
468 
469     errors.print("Average load: " + status.getAverageLoad());
470     errors.print("Number of requests: " + status.getRequestsCount());
471     errors.print("Number of regions: " + status.getRegionsCount());
472 
473     Map<String, RegionState> rits = status.getRegionsInTransition();
474     errors.print("Number of regions in transition: " + rits.size());
475     if (details) {
476       for (RegionState state: rits.values()) {
477         errors.print("  " + state.toDescriptiveString());
478       }
479     }
480 
481     // Determine what's deployed
482     processRegionServers(regionServers);
483   }
484 
485   /**
486    * Clear the current state of hbck.
487    */
488   private void clearState() {
489     // Make sure regionInfo is empty before starting
490     fixes = 0;
491     regionInfoMap.clear();
492     emptyRegionInfoQualifiers.clear();
493     disabledTables.clear();
494     errors.clear();
495     tablesInfo.clear();
496     orphanHdfsDirs.clear();
497   }
498 
499   /**
500    * This repair method analyzes hbase data in hdfs and repairs it to satisfy
501    * the table integrity rules.  HBase doesn't need to be online for this
502    * operation to work.
503    */
504   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
505     // Initial pass to fix orphans.
506     if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
507         || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
508       LOG.info("Loading regioninfos HDFS");
509       // if nothing is happening this should always complete in two iterations.
510       int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
511       int curIter = 0;
512       do {
513         clearState(); // clears hbck state and reset fixes to 0 and.
514         // repair what's on HDFS
515         restoreHdfsIntegrity();
516         curIter++;// limit the number of iterations.
517       } while (fixes > 0 && curIter <= maxIterations);
518 
519       // Repairs should be done in the first iteration and verification in the second.
520       // If there are more than 2 passes, something funny has happened.
521       if (curIter > 2) {
522         if (curIter == maxIterations) {
523           LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
524               + "Tables integrity may not be fully repaired!");
525         } else {
526           LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
527         }
528       }
529     }
530   }
531 
532   /**
533    * This repair method requires the cluster to be online since it contacts
534    * region servers and the masters.  It makes each region's state in HDFS, in
535    * hbase:meta, and deployments consistent.
536    *
537    * @return If > 0 , number of errors detected, if < 0 there was an unrecoverable
538    * error.  If 0, we have a clean hbase.
539    */
540   public int onlineConsistencyRepair() throws IOException, KeeperException,
541     InterruptedException {
542     clearState();
543 
544     // get regions according to what is online on each RegionServer
545     loadDeployedRegions();
546     // check whether hbase:meta is deployed and online
547     recordMetaRegion();
548     // Check if hbase:meta is found only once and in the right place
549     if (!checkMetaRegion()) {
550       String errorMsg = "hbase:meta table is not consistent. ";
551       if (shouldFixAssignments()) {
552         errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
553       } else {
554         errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
555       }
556       errors.reportError(errorMsg + " Exiting...");
557       return -2;
558     }
559     // Not going with further consistency check for tables when hbase:meta itself is not consistent.
560     LOG.info("Loading regionsinfo from the hbase:meta table");
561     boolean success = loadMetaEntries();
562     if (!success) return -1;
563 
564     // Empty cells in hbase:meta?
565     reportEmptyMetaCells();
566 
567     // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
568     if (shouldFixEmptyMetaCells()) {
569       fixEmptyMetaCells();
570     }
571 
572     // get a list of all tables that have not changed recently.
573     if (!checkMetaOnly) {
574       reportTablesInFlux();
575     }
576 
577     // load regiondirs and regioninfos from HDFS
578     if (shouldCheckHdfs()) {
579       loadHdfsRegionDirs();
580       loadHdfsRegionInfos();
581     }
582 
583     // Get disabled tables from ZooKeeper
584     loadDisabledTables();
585 
586     // fix the orphan tables
587     fixOrphanTables();
588 
589     // Check and fix consistency
590     checkAndFixConsistency();
591 
592     // Check integrity (does not fix)
593     checkIntegrity();
594     return errors.getErrorList().size();
595   }
596 
597   /**
598    * Contacts the master and prints out cluster-wide information
599    * @return 0 on success, non-zero on failure
600    */
601   public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
602     // print hbase server version
603     errors.print("Version: " + status.getHBaseVersion());
604     offlineHdfsIntegrityRepair();
605 
606     // turn the balancer off
607     boolean oldBalancer = admin.setBalancerRunning(false, true);
608     try {
609       onlineConsistencyRepair();
610     }
611     finally {
612       admin.setBalancerRunning(oldBalancer, false);
613     }
614 
615     if (checkRegionBoundaries) {
616       checkRegionBoundaries();
617     }
618 
619     offlineReferenceFileRepair();
620 
621     checkAndFixTableLocks();
622 
623     // Remove the hbck lock
624     unlockHbck();
625 
626     // Print table summary
627     printTableSummary(tablesInfo);
628     return errors.summarize();
629   }
630 
631   public static byte[] keyOnly (byte[] b) {
632     if (b == null)
633       return b;
634     int rowlength = Bytes.toShort(b, 0);
635     byte[] result = new byte[rowlength];
636     System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
637     return result;
638   }
639 
640   @Override
641   public void close() throws IOException {
642     IOUtils.cleanup(null, admin, meta, connection);
643   }
644 
645   private static class RegionBoundariesInformation {
646     public byte [] regionName;
647     public byte [] metaFirstKey;
648     public byte [] metaLastKey;
649     public byte [] storesFirstKey;
650     public byte [] storesLastKey;
651     @Override
652     public String toString () {
653       return "regionName=" + Bytes.toStringBinary(regionName) +
654              "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
655              "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
656              "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
657              "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
658     }
659   }
660 
661   public void checkRegionBoundaries() {
662     try {
663       ByteArrayComparator comparator = new ByteArrayComparator();
664       List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), connection, false);
665       final RegionBoundariesInformation currentRegionBoundariesInformation =
666           new RegionBoundariesInformation();
667       Path hbaseRoot = FSUtils.getRootDir(getConf());
668       for (HRegionInfo regionInfo : regions) {
669         Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
670         currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
671         // For each region, get the start and stop key from the META and compare them to the
672         // same information from the Stores.
673         Path path = new Path(tableDir, regionInfo.getEncodedName());
674         FileSystem fs = path.getFileSystem(getConf());
675         FileStatus[] files = fs.listStatus(path);
676         // For all the column families in this region...
677         byte[] storeFirstKey = null;
678         byte[] storeLastKey = null;
679         for (FileStatus file : files) {
680           String fileName = file.getPath().toString();
681           fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
682           if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
683             FileStatus[] storeFiles = fs.listStatus(file.getPath());
684             // For all the stores in this column family.
685             for (FileStatus storeFile : storeFiles) {
686               HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
687                   getConf()), getConf());
688               if ((reader.getFirstKey() != null)
689                   && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
690                       reader.getFirstKey()) > 0))) {
691                 storeFirstKey = reader.getFirstKey();
692               }
693               if ((reader.getLastKey() != null)
694                   && ((storeLastKey == null) || (comparator.compare(storeLastKey,
695                       reader.getLastKey())) < 0)) {
696                 storeLastKey = reader.getLastKey();
697               }
698               reader.close();
699             }
700           }
701         }
702         currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
703         currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
704         currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
705         currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
706         if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
707           currentRegionBoundariesInformation.metaFirstKey = null;
708         if (currentRegionBoundariesInformation.metaLastKey.length == 0)
709           currentRegionBoundariesInformation.metaLastKey = null;
710 
711         // For a region to be correct, we need the META start key to be smaller or equal to the
712         // smallest start key from all the stores, and the start key from the next META entry to
713         // be bigger than the last key from all the current stores. First region start key is null;
714         // Last region end key is null; some regions can be empty and not have any store.
715 
716         boolean valid = true;
717         // Checking start key.
718         if ((currentRegionBoundariesInformation.storesFirstKey != null)
719             && (currentRegionBoundariesInformation.metaFirstKey != null)) {
720           valid = valid
721               && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
722                 currentRegionBoundariesInformation.metaFirstKey) >= 0;
723         }
724         // Checking stop key.
725         if ((currentRegionBoundariesInformation.storesLastKey != null)
726             && (currentRegionBoundariesInformation.metaLastKey != null)) {
727           valid = valid
728               && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
729                 currentRegionBoundariesInformation.metaLastKey) < 0;
730         }
731         if (!valid) {
732           errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
733             tablesInfo.get(regionInfo.getTable()));
734           LOG.warn("Region's boundaries not alligned between stores and META for:");
735           LOG.warn(currentRegionBoundariesInformation);
736         }
737       }
738     } catch (IOException e) {
739       LOG.error(e);
740     }
741   }
742 
743   /**
744    * Iterates through the list of all orphan/invalid regiondirs.
745    */
746   private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
747     for (HbckInfo hi : orphanHdfsDirs) {
748       LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
749       adoptHdfsOrphan(hi);
750     }
751   }
752 
753   /**
754    * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
755    * these orphans by creating a new region, and moving the column families,
756    * recovered edits, WALs, into the new region dir.  We determine the region
757    * startkey and endkeys by looking at all of the hfiles inside the column
758    * families to identify the min and max keys. The resulting region will
759    * likely violate table integrity but will be dealt with by merging
760    * overlapping regions.
761    */
762   @SuppressWarnings("deprecation")
763   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
764     Path p = hi.getHdfsRegionDir();
765     FileSystem fs = p.getFileSystem(getConf());
766     FileStatus[] dirs = fs.listStatus(p);
767     if (dirs == null) {
768       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
769           p + ". This dir could probably be deleted.");
770       return ;
771     }
772 
773     TableName tableName = hi.getTableName();
774     TableInfo tableInfo = tablesInfo.get(tableName);
775     Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
776     HTableDescriptor template = tableInfo.getHTD();
777 
778     // find min and max key values
779     Pair<byte[],byte[]> orphanRegionRange = null;
780     for (FileStatus cf : dirs) {
781       String cfName= cf.getPath().getName();
782       // TODO Figure out what the special dirs are
783       if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
784 
785       FileStatus[] hfiles = fs.listStatus(cf.getPath());
786       for (FileStatus hfile : hfiles) {
787         byte[] start, end;
788         HFile.Reader hf = null;
789         try {
790           CacheConfig cacheConf = new CacheConfig(getConf());
791           hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
792           hf.loadFileInfo();
793           KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
794           start = startKv.getRow();
795           KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
796           end = endKv.getRow();
797         } catch (IOException ioe) {
798           LOG.warn("Problem reading orphan file " + hfile + ", skipping");
799           continue;
800         } catch (NullPointerException ioe) {
801           LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
802           continue;
803         } finally {
804           if (hf != null) {
805             hf.close();
806           }
807         }
808 
809         // expand the range to include the range of all hfiles
810         if (orphanRegionRange == null) {
811           // first range
812           orphanRegionRange = new Pair<byte[], byte[]>(start, end);
813         } else {
814           // TODO add test
815 
816           // expand range only if the hfile is wider.
817           if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
818             orphanRegionRange.setFirst(start);
819           }
820           if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
821             orphanRegionRange.setSecond(end);
822           }
823         }
824       }
825     }
826     if (orphanRegionRange == null) {
827       LOG.warn("No data in dir " + p + ", sidelining data");
828       fixes++;
829       sidelineRegionDir(fs, hi);
830       return;
831     }
832     LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
833         Bytes.toString(orphanRegionRange.getSecond()) + ")");
834 
835     // create new region on hdfs.  move data into place.
836     HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond());
837     LOG.info("Creating new region : " + hri);
838     HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
839     Path target = region.getRegionFileSystem().getRegionDir();
840 
841     // rename all the data to new region
842     mergeRegionDirs(target, hi);
843     fixes++;
844   }
845 
846   /**
847    * This method determines if there are table integrity errors in HDFS.  If
848    * there are errors and the appropriate "fix" options are enabled, the method
849    * will first correct orphan regions making them into legit regiondirs, and
850    * then reload to merge potentially overlapping regions.
851    *
852    * @return number of table integrity errors found
853    */
854   private int restoreHdfsIntegrity() throws IOException, InterruptedException {
855     // Determine what's on HDFS
856     LOG.info("Loading HBase regioninfo from HDFS...");
857     loadHdfsRegionDirs(); // populating regioninfo table.
858 
859     int errs = errors.getErrorList().size();
860     // First time just get suggestions.
861     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
862     checkHdfsIntegrity(false, false);
863 
864     if (errors.getErrorList().size() == errs) {
865       LOG.info("No integrity errors.  We are done with this phase. Glorious.");
866       return 0;
867     }
868 
869     if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
870       adoptHdfsOrphans(orphanHdfsDirs);
871       // TODO optimize by incrementally adding instead of reloading.
872     }
873 
874     // Make sure there are no holes now.
875     if (shouldFixHdfsHoles()) {
876       clearState(); // this also resets # fixes.
877       loadHdfsRegionDirs();
878       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
879       tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
880     }
881 
882     // Now we fix overlaps
883     if (shouldFixHdfsOverlaps()) {
884       // second pass we fix overlaps.
885       clearState(); // this also resets # fixes.
886       loadHdfsRegionDirs();
887       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
888       tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
889     }
890 
891     return errors.getErrorList().size();
892   }
893 
894   /**
895    * Scan all the store file names to find any lingering reference files,
896    * which refer to some none-exiting files. If "fix" option is enabled,
897    * any lingering reference file will be sidelined if found.
898    * <p>
899    * Lingering reference file prevents a region from opening. It has to
900    * be fixed before a cluster can start properly.
901    */
902   private void offlineReferenceFileRepair() throws IOException {
903     Configuration conf = getConf();
904     Path hbaseRoot = FSUtils.getRootDir(conf);
905     FileSystem fs = hbaseRoot.getFileSystem(conf);
906     Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot);
907     for (Path path: allFiles.values()) {
908       boolean isReference = false;
909       try {
910         isReference = StoreFileInfo.isReference(path);
911       } catch (Throwable t) {
912         // Ignore. Some files may not be store files at all.
913         // For example, files under .oldlogs folder in hbase:meta
914         // Warning message is already logged by
915         // StoreFile#isReference.
916       }
917       if (!isReference) continue;
918 
919       Path referredToFile = StoreFileInfo.getReferredToFile(path);
920       if (fs.exists(referredToFile)) continue;  // good, expected
921 
922       // Found a lingering reference file
923       errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
924         "Found lingering reference file " + path);
925       if (!shouldFixReferenceFiles()) continue;
926 
927       // Now, trying to fix it since requested
928       boolean success = false;
929       String pathStr = path.toString();
930 
931       // A reference file path should be like
932       // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
933       // Up 5 directories to get the root folder.
934       // So the file will be sidelined to a similar folder structure.
935       int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
936       for (int i = 0; index > 0 && i < 5; i++) {
937         index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
938       }
939       if (index > 0) {
940         Path rootDir = getSidelineDir();
941         Path dst = new Path(rootDir, pathStr.substring(index + 1));
942         fs.mkdirs(dst.getParent());
943         LOG.info("Trying to sildeline reference file "
944           + path + " to " + dst);
945         setShouldRerun();
946 
947         success = fs.rename(path, dst);
948       }
949       if (!success) {
950         LOG.error("Failed to sideline reference file " + path);
951       }
952     }
953   }
954 
955   /**
956    * TODO -- need to add tests for this.
957    */
958   private void reportEmptyMetaCells() {
959     errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
960       emptyRegionInfoQualifiers.size());
961     if (details) {
962       for (Result r: emptyRegionInfoQualifiers) {
963         errors.print("  " + r);
964       }
965     }
966   }
967 
968   /**
969    * TODO -- need to add tests for this.
970    */
971   private void reportTablesInFlux() {
972     AtomicInteger numSkipped = new AtomicInteger(0);
973     HTableDescriptor[] allTables = getTables(numSkipped);
974     errors.print("Number of Tables: " + allTables.length);
975     if (details) {
976       if (numSkipped.get() > 0) {
977         errors.detail("Number of Tables in flux: " + numSkipped.get());
978       }
979       for (HTableDescriptor td : allTables) {
980         errors.detail("  Table: " + td.getTableName() + "\t" +
981                            (td.isReadOnly() ? "ro" : "rw") + "\t" +
982                             (td.isMetaRegion() ? "META" : "    ") + "\t" +
983                            " families: " + td.getFamilies().size());
984       }
985     }
986   }
987 
988   public ErrorReporter getErrors() {
989     return errors;
990   }
991 
992   /**
993    * Read the .regioninfo file from the file system.  If there is no
994    * .regioninfo, add it to the orphan hdfs region list.
995    */
996   private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
997     Path regionDir = hbi.getHdfsRegionDir();
998     if (regionDir == null) {
999       LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
1000       return;
1001     }
1002 
1003     if (hbi.hdfsEntry.hri != null) {
1004       // already loaded data
1005       return;
1006     }
1007 
1008     FileSystem fs = FileSystem.get(getConf());
1009     HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
1010     LOG.debug("HRegionInfo read: " + hri.toString());
1011     hbi.hdfsEntry.hri = hri;
1012   }
1013 
1014   /**
1015    * Exception thrown when a integrity repair operation fails in an
1016    * unresolvable way.
1017    */
1018   public static class RegionRepairException extends IOException {
1019     private static final long serialVersionUID = 1L;
1020     final IOException ioe;
1021     public RegionRepairException(String s, IOException ioe) {
1022       super(s);
1023       this.ioe = ioe;
1024     }
1025   }
1026 
1027   /**
1028    * Populate hbi's from regionInfos loaded from file system.
1029    */
1030   private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
1031       throws IOException, InterruptedException {
1032     tablesInfo.clear(); // regenerating the data
1033     // generate region split structure
1034     Collection<HbckInfo> hbckInfos = regionInfoMap.values();
1035 
1036     // Parallelized read of .regioninfo files.
1037     List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
1038     List<Future<Void>> hbiFutures;
1039 
1040     for (HbckInfo hbi : hbckInfos) {
1041       WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1042       hbis.add(work);
1043     }
1044 
1045     // Submit and wait for completion
1046     hbiFutures = executor.invokeAll(hbis);
1047 
1048     for(int i=0; i<hbiFutures.size(); i++) {
1049       WorkItemHdfsRegionInfo work = hbis.get(i);
1050       Future<Void> f = hbiFutures.get(i);
1051       try {
1052         f.get();
1053       } catch(ExecutionException e) {
1054         LOG.warn("Failed to read .regioninfo file for region " +
1055               work.hbi.getRegionNameAsString(), e.getCause());
1056       }
1057     }
1058 
1059     Path hbaseRoot = FSUtils.getRootDir(getConf());
1060     FileSystem fs = hbaseRoot.getFileSystem(getConf());
1061     // serialized table info gathering.
1062     for (HbckInfo hbi: hbckInfos) {
1063 
1064       if (hbi.getHdfsHRI() == null) {
1065         // was an orphan
1066         continue;
1067       }
1068 
1069 
1070       // get table name from hdfs, populate various HBaseFsck tables.
1071       TableName tableName = hbi.getTableName();
1072       if (tableName == null) {
1073         // There was an entry in hbase:meta not in the HDFS?
1074         LOG.warn("tableName was null for: " + hbi);
1075         continue;
1076       }
1077 
1078       TableInfo modTInfo = tablesInfo.get(tableName);
1079       if (modTInfo == null) {
1080         // only executed once per table.
1081         modTInfo = new TableInfo(tableName);
1082         tablesInfo.put(tableName, modTInfo);
1083         try {
1084           TableDescriptor htd =
1085               FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1086           modTInfo.htds.add(htd.getHTableDescriptor());
1087         } catch (IOException ioe) {
1088           if (!orphanTableDirs.containsKey(tableName)) {
1089             LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1090             //should only report once for each table
1091             errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1092                 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1093             Set<String> columns = new HashSet<String>();
1094             orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1095           }
1096         }
1097       }
1098       if (!hbi.isSkipChecks()) {
1099         modTInfo.addRegionInfo(hbi);
1100       }
1101     }
1102 
1103     loadTableInfosForTablesWithNoRegion();
1104 
1105     return tablesInfo;
1106   }
1107 
1108   /**
1109    * To get the column family list according to the column family dirs
1110    * @param columns
1111    * @param hbi
1112    * @return a set of column families
1113    * @throws IOException
1114    */
1115   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
1116     Path regionDir = hbi.getHdfsRegionDir();
1117     FileSystem fs = regionDir.getFileSystem(getConf());
1118     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1119     for (FileStatus subdir : subDirs) {
1120       String columnfamily = subdir.getPath().getName();
1121       columns.add(columnfamily);
1122     }
1123     return columns;
1124   }
1125 
1126   /**
1127    * To fabricate a .tableinfo file with following contents<br>
1128    * 1. the correct tablename <br>
1129    * 2. the correct colfamily list<br>
1130    * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1131    * @throws IOException
1132    */
1133   private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1134       Set<String> columns) throws IOException {
1135     if (columns ==null || columns.isEmpty()) return false;
1136     HTableDescriptor htd = new HTableDescriptor(tableName);
1137     for (String columnfamimly : columns) {
1138       htd.addFamily(new HColumnDescriptor(columnfamimly));
1139     }
1140     fstd.createTableDescriptor(new TableDescriptor(htd, TableState.State.ENABLED), true);
1141     return true;
1142   }
1143 
1144   /**
1145    * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1146    * @throws IOException
1147    */
1148   public void fixEmptyMetaCells() throws IOException {
1149     if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1150       LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1151       for (Result region : emptyRegionInfoQualifiers) {
1152         deleteMetaRegion(region.getRow());
1153         errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1154       }
1155       emptyRegionInfoQualifiers.clear();
1156     }
1157   }
1158 
1159   /**
1160    * To fix orphan table by creating a .tableinfo file under tableDir <br>
1161    * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1162    * 2. else create a default .tableinfo file with following items<br>
1163    * &nbsp;2.1 the correct tablename <br>
1164    * &nbsp;2.2 the correct colfamily list<br>
1165    * &nbsp;2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1166    * @throws IOException
1167    */
1168   public void fixOrphanTables() throws IOException {
1169     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1170 
1171       List<TableName> tmpList = new ArrayList<TableName>();
1172       tmpList.addAll(orphanTableDirs.keySet());
1173       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1174       Iterator<Entry<TableName, Set<String>>> iter =
1175           orphanTableDirs.entrySet().iterator();
1176       int j = 0;
1177       int numFailedCase = 0;
1178       FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1179       while (iter.hasNext()) {
1180         Entry<TableName, Set<String>> entry =
1181             iter.next();
1182         TableName tableName = entry.getKey();
1183         LOG.info("Trying to fix orphan table error: " + tableName);
1184         if (j < htds.length) {
1185           if (tableName.equals(htds[j].getTableName())) {
1186             HTableDescriptor htd = htds[j];
1187             LOG.info("fixing orphan table: " + tableName + " from cache");
1188             fstd.createTableDescriptor(new TableDescriptor(htd, TableState.State.ENABLED), true);
1189             j++;
1190             iter.remove();
1191           }
1192         } else {
1193           if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1194             LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1195             LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1196             iter.remove();
1197           } else {
1198             LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1199             numFailedCase++;
1200           }
1201         }
1202         fixes++;
1203       }
1204 
1205       if (orphanTableDirs.isEmpty()) {
1206         // all orphanTableDirs are luckily recovered
1207         // re-run doFsck after recovering the .tableinfo file
1208         setShouldRerun();
1209         LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1210       } else if (numFailedCase > 0) {
1211         LOG.error("Failed to fix " + numFailedCase
1212             + " OrphanTables with default .tableinfo files");
1213       }
1214 
1215     }
1216     //cleanup the list
1217     orphanTableDirs.clear();
1218 
1219   }
1220 
1221   /**
1222    * This borrows code from MasterFileSystem.bootstrap(). Explicitly creates it's own WAL, so be
1223    * sure to close it as well as the region when you're finished.
1224    *
1225    * @return an open hbase:meta HRegion
1226    */
1227   private HRegion createNewMeta() throws IOException {
1228     Path rootdir = FSUtils.getRootDir(getConf());
1229     Configuration c = getConf();
1230     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1231     HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
1232     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false);
1233     // The WAL subsystem will use the default rootDir rather than the passed in rootDir
1234     // unless I pass along via the conf.
1235     Configuration confForWAL = new Configuration(c);
1236     confForWAL.set(HConstants.HBASE_DIR, rootdir.toString());
1237     WAL wal = (new WALFactory(confForWAL,
1238         Collections.<WALActionsListener>singletonList(new MetricsWAL()),
1239         "hbck-meta-recovery-" + RandomStringUtils.randomNumeric(8))).
1240         getWAL(metaHRI.getEncodedNameAsBytes());
1241     HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor, wal);
1242     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
1243     return meta;
1244   }
1245 
1246   /**
1247    * Generate set of puts to add to new meta.  This expects the tables to be
1248    * clean with no overlaps or holes.  If there are any problems it returns null.
1249    *
1250    * @return An array list of puts to do in bulk, null if tables have problems
1251    */
1252   private ArrayList<Put> generatePuts(
1253       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1254     ArrayList<Put> puts = new ArrayList<Put>();
1255     boolean hasProblems = false;
1256     for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1257       TableName name = e.getKey();
1258 
1259       // skip "hbase:meta"
1260       if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1261         continue;
1262       }
1263 
1264       TableInfo ti = e.getValue();
1265       for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1266           .entrySet()) {
1267         Collection<HbckInfo> his = spl.getValue();
1268         int sz = his.size();
1269         if (sz != 1) {
1270           // problem
1271           LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1272               + " had " +  sz + " regions instead of exactly 1." );
1273           hasProblems = true;
1274           continue;
1275         }
1276 
1277         // add the row directly to meta.
1278         HbckInfo hi = his.iterator().next();
1279         HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
1280         Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1281         puts.add(p);
1282       }
1283     }
1284     return hasProblems ? null : puts;
1285   }
1286 
1287   /**
1288    * Suggest fixes for each table
1289    */
1290   private void suggestFixes(
1291       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1292     for (TableInfo tInfo : tablesInfo.values()) {
1293       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1294       tInfo.checkRegionChain(handler);
1295     }
1296   }
1297 
1298   /**
1299    * Rebuilds meta from information in hdfs/fs.  Depends on configuration settings passed into
1300    * hbck constructor to point to a particular fs/dir. Assumes HBase is OFFLINE.
1301    *
1302    * @param fix flag that determines if method should attempt to fix holes
1303    * @return true if successful, false if attempt failed.
1304    */
1305   public boolean rebuildMeta(boolean fix) throws IOException,
1306       InterruptedException {
1307 
1308     // TODO check to make sure hbase is offline. (or at least the table
1309     // currently being worked on is off line)
1310 
1311     // Determine what's on HDFS
1312     LOG.info("Loading HBase regioninfo from HDFS...");
1313     loadHdfsRegionDirs(); // populating regioninfo table.
1314 
1315     int errs = errors.getErrorList().size();
1316     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1317     checkHdfsIntegrity(false, false);
1318 
1319     // make sure ok.
1320     if (errors.getErrorList().size() != errs) {
1321       // While in error state, iterate until no more fixes possible
1322       while(true) {
1323         fixes = 0;
1324         suggestFixes(tablesInfo);
1325         errors.clear();
1326         loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1327         checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1328 
1329         int errCount = errors.getErrorList().size();
1330 
1331         if (fixes == 0) {
1332           if (errCount > 0) {
1333             return false; // failed to fix problems.
1334           } else {
1335             break; // no fixes and no problems? drop out and fix stuff!
1336           }
1337         }
1338       }
1339     }
1340 
1341     // we can rebuild, move old meta out of the way and start
1342     LOG.info("HDFS regioninfo's seems good.  Sidelining old hbase:meta");
1343     Path backupDir = sidelineOldMeta();
1344 
1345     LOG.info("Creating new hbase:meta");
1346     HRegion meta = createNewMeta();
1347 
1348     // populate meta
1349     List<Put> puts = generatePuts(tablesInfo);
1350     if (puts == null) {
1351       LOG.fatal("Problem encountered when creating new hbase:meta entries.  " +
1352         "You may need to restore the previously sidelined hbase:meta");
1353       return false;
1354     }
1355     meta.batchMutate(puts.toArray(new Put[puts.size()]));
1356     meta.close();
1357     if (meta.getWAL() != null) {
1358       meta.getWAL().close();
1359     }
1360     LOG.info("Success! hbase:meta table rebuilt.");
1361     LOG.info("Old hbase:meta is moved into " + backupDir);
1362     return true;
1363   }
1364 
1365   private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1366       boolean fixOverlaps) throws IOException {
1367     LOG.info("Checking HBase region split map from HDFS data...");
1368     for (TableInfo tInfo : tablesInfo.values()) {
1369       TableIntegrityErrorHandler handler;
1370       if (fixHoles || fixOverlaps) {
1371         handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1372           fixHoles, fixOverlaps);
1373       } else {
1374         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1375       }
1376       if (!tInfo.checkRegionChain(handler)) {
1377         // should dump info as well.
1378         errors.report("Found inconsistency in table " + tInfo.getName());
1379       }
1380     }
1381     return tablesInfo;
1382   }
1383 
1384   private Path getSidelineDir() throws IOException {
1385     if (sidelineDir == null) {
1386       Path hbaseDir = FSUtils.getRootDir(getConf());
1387       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1388       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1389           + startMillis);
1390     }
1391     return sidelineDir;
1392   }
1393 
1394   /**
1395    * Sideline a region dir (instead of deleting it)
1396    */
1397   Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1398     return sidelineRegionDir(fs, null, hi);
1399   }
1400 
1401   /**
1402    * Sideline a region dir (instead of deleting it)
1403    *
1404    * @param parentDir if specified, the region will be sidelined to
1405    * folder like .../parentDir/<table name>/<region name>. The purpose
1406    * is to group together similar regions sidelined, for example, those
1407    * regions should be bulk loaded back later on. If null, it is ignored.
1408    */
1409   Path sidelineRegionDir(FileSystem fs,
1410       String parentDir, HbckInfo hi) throws IOException {
1411     TableName tableName = hi.getTableName();
1412     Path regionDir = hi.getHdfsRegionDir();
1413 
1414     if (!fs.exists(regionDir)) {
1415       LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1416       return null;
1417     }
1418 
1419     Path rootDir = getSidelineDir();
1420     if (parentDir != null) {
1421       rootDir = new Path(rootDir, parentDir);
1422     }
1423     Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1424     Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1425     fs.mkdirs(sidelineRegionDir);
1426     boolean success = false;
1427     FileStatus[] cfs =  fs.listStatus(regionDir);
1428     if (cfs == null) {
1429       LOG.info("Region dir is empty: " + regionDir);
1430     } else {
1431       for (FileStatus cf : cfs) {
1432         Path src = cf.getPath();
1433         Path dst =  new Path(sidelineRegionDir, src.getName());
1434         if (fs.isFile(src)) {
1435           // simple file
1436           success = fs.rename(src, dst);
1437           if (!success) {
1438             String msg = "Unable to rename file " + src +  " to " + dst;
1439             LOG.error(msg);
1440             throw new IOException(msg);
1441           }
1442           continue;
1443         }
1444 
1445         // is a directory.
1446         fs.mkdirs(dst);
1447 
1448         LOG.info("Sidelining files from " + src + " into containing region " + dst);
1449         // FileSystem.rename is inconsistent with directories -- if the
1450         // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1451         // it moves the src into the dst dir resulting in (foo/a/b).  If
1452         // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1453         FileStatus[] hfiles = fs.listStatus(src);
1454         if (hfiles != null && hfiles.length > 0) {
1455           for (FileStatus hfile : hfiles) {
1456             success = fs.rename(hfile.getPath(), dst);
1457             if (!success) {
1458               String msg = "Unable to rename file " + src +  " to " + dst;
1459               LOG.error(msg);
1460               throw new IOException(msg);
1461             }
1462           }
1463         }
1464         LOG.debug("Sideline directory contents:");
1465         debugLsr(sidelineRegionDir);
1466       }
1467     }
1468 
1469     LOG.info("Removing old region dir: " + regionDir);
1470     success = fs.delete(regionDir, true);
1471     if (!success) {
1472       String msg = "Unable to delete dir " + regionDir;
1473       LOG.error(msg);
1474       throw new IOException(msg);
1475     }
1476     return sidelineRegionDir;
1477   }
1478 
1479   /**
1480    * Side line an entire table.
1481    */
1482   void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1483       Path backupHbaseDir) throws IOException {
1484     Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1485     if (fs.exists(tableDir)) {
1486       Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1487       fs.mkdirs(backupTableDir.getParent());
1488       boolean success = fs.rename(tableDir, backupTableDir);
1489       if (!success) {
1490         throw new IOException("Failed to move  " + tableName + " from "
1491             +  tableDir + " to " + backupTableDir);
1492       }
1493     } else {
1494       LOG.info("No previous " + tableName +  " exists.  Continuing.");
1495     }
1496   }
1497 
1498   /**
1499    * @return Path to backup of original directory
1500    */
1501   Path sidelineOldMeta() throws IOException {
1502     // put current hbase:meta aside.
1503     Path hbaseDir = FSUtils.getRootDir(getConf());
1504     FileSystem fs = hbaseDir.getFileSystem(getConf());
1505     Path backupDir = getSidelineDir();
1506     fs.mkdirs(backupDir);
1507 
1508     try {
1509       sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1510     } catch (IOException e) {
1511         LOG.fatal("... failed to sideline meta. Currently in inconsistent state.  To restore "
1512             + "try to rename hbase:meta in " + backupDir.getName() + " to "
1513             + hbaseDir.getName() + ".", e);
1514       throw e; // throw original exception
1515     }
1516     return backupDir;
1517   }
1518 
1519   /**
1520    * Load the list of disabled tables in ZK into local set.
1521    * @throws ZooKeeperConnectionException
1522    * @throws IOException
1523    */
1524   private void loadDisabledTables()
1525   throws IOException {
1526     HConnectionManager.execute(new HConnectable<Void>(getConf()) {
1527       @Override
1528       public Void connect(HConnection connection) throws IOException {
1529         TableName[] tables = connection.listTableNames();
1530         for (TableName table : tables) {
1531           if (connection.getTableState(table)
1532               .inStates(TableState.State.DISABLED, TableState.State.DISABLING)) {
1533             disabledTables.add(table);
1534           }
1535         }
1536         return null;
1537       }
1538     });
1539   }
1540 
1541   /**
1542    * Check if the specified region's table is disabled.
1543    */
1544   private boolean isTableDisabled(HRegionInfo regionInfo) {
1545     return disabledTables.contains(regionInfo.getTable());
1546   }
1547 
1548   /**
1549    * Scan HDFS for all regions, recording their information into
1550    * regionInfoMap
1551    */
1552   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1553     Path rootDir = FSUtils.getRootDir(getConf());
1554     FileSystem fs = rootDir.getFileSystem(getConf());
1555 
1556     // list all tables from HDFS
1557     List<FileStatus> tableDirs = Lists.newArrayList();
1558 
1559     boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1560 
1561     List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1562     for (Path path : paths) {
1563       TableName tableName = FSUtils.getTableName(path);
1564        if ((!checkMetaOnly &&
1565            isTableIncluded(tableName)) ||
1566            tableName.equals(TableName.META_TABLE_NAME)) {
1567          tableDirs.add(fs.getFileStatus(path));
1568        }
1569     }
1570 
1571     // verify that version file exists
1572     if (!foundVersionFile) {
1573       errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1574           "Version file does not exist in root dir " + rootDir);
1575       if (shouldFixVersionFile()) {
1576         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1577             + " file.");
1578         setShouldRerun();
1579         FSUtils.setVersion(fs, rootDir, getConf().getInt(
1580             HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1581             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1582             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1583       }
1584     }
1585 
1586     // level 1:  <HBASE_DIR>/*
1587     List<WorkItemHdfsDir> dirs = new ArrayList<WorkItemHdfsDir>(tableDirs.size());
1588     List<Future<Void>> dirsFutures;
1589 
1590     for (FileStatus tableDir : tableDirs) {
1591       LOG.debug("Loading region dirs from " +tableDir.getPath());
1592       dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir));
1593     }
1594 
1595     // Invoke and wait for Callables to complete
1596     dirsFutures = executor.invokeAll(dirs);
1597 
1598     for(Future<Void> f: dirsFutures) {
1599       try {
1600         f.get();
1601       } catch(ExecutionException e) {
1602         LOG.warn("Could not load region dir " , e.getCause());
1603       }
1604     }
1605   }
1606 
1607   /**
1608    * Record the location of the hbase:meta region as found in ZooKeeper.
1609    */
1610   private boolean recordMetaRegion() throws IOException {
1611     HRegionLocation metaLocation = connection.locateRegion(
1612       TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW);
1613 
1614     // Check if Meta region is valid and existing
1615     if (metaLocation == null || metaLocation.getRegionInfo() == null ||
1616         metaLocation.getHostname() == null) {
1617       errors.reportError(ERROR_CODE.NULL_META_REGION,
1618         "META region or some of its attributes are null.");
1619       return false;
1620     }
1621     ServerName sn;
1622     try {
1623       sn = getMetaRegionServerName();
1624     } catch (KeeperException e) {
1625       throw new IOException(e);
1626     }
1627     MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, System.currentTimeMillis());
1628     HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1629     if (hbckInfo == null) {
1630       regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1631     } else {
1632       hbckInfo.metaEntry = m;
1633     }
1634     return true;
1635   }
1636 
1637   private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1638     return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1639       @Override
1640       public void abort(String why, Throwable e) {
1641         LOG.error(why, e);
1642         System.exit(1);
1643       }
1644 
1645       @Override
1646       public boolean isAborted() {
1647         return false;
1648       }
1649 
1650     });
1651   }
1652 
1653   private ServerName getMetaRegionServerName()
1654   throws IOException, KeeperException {
1655     ZooKeeperWatcher zkw = createZooKeeperWatcher();
1656     ServerName sn = null;
1657     try {
1658       sn = new MetaTableLocator().getMetaRegionLocation(zkw);
1659     } finally {
1660       zkw.close();
1661     }
1662     return sn;
1663   }
1664 
1665   /**
1666    * Contacts each regionserver and fetches metadata about regions.
1667    * @param regionServerList - the list of region servers to connect to
1668    * @throws IOException if a remote or network exception occurs
1669    */
1670   void processRegionServers(Collection<ServerName> regionServerList)
1671     throws IOException, InterruptedException {
1672 
1673     List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1674     List<Future<Void>> workFutures;
1675 
1676     // loop to contact each region server in parallel
1677     for (ServerName rsinfo: regionServerList) {
1678       workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1679     }
1680 
1681     workFutures = executor.invokeAll(workItems);
1682 
1683     for(int i=0; i<workFutures.size(); i++) {
1684       WorkItemRegion item = workItems.get(i);
1685       Future<Void> f = workFutures.get(i);
1686       try {
1687         f.get();
1688       } catch(ExecutionException e) {
1689         LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1690             e.getCause());
1691       }
1692     }
1693   }
1694 
1695   /**
1696    * Check consistency of all regions that have been found in previous phases.
1697    */
1698   private void checkAndFixConsistency()
1699   throws IOException, KeeperException, InterruptedException {
1700     // Divide the checks in two phases. One for default/primary replicas and another
1701     // for the non-primary ones. Keeps code cleaner this way.
1702     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1703       if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1704         checkRegionConsistency(e.getKey(), e.getValue());
1705       }
1706     }
1707     boolean prevHdfsCheck = shouldCheckHdfs();
1708     setCheckHdfs(false); //replicas don't have any hdfs data
1709     // Run a pass over the replicas and fix any assignment issues that exist on the currently
1710     // deployed/undeployed replicas.
1711     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1712       if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1713         checkRegionConsistency(e.getKey(), e.getValue());
1714       }
1715     }
1716     setCheckHdfs(prevHdfsCheck);
1717   }
1718 
1719   private void preCheckPermission() throws IOException, AccessDeniedException {
1720     if (shouldIgnorePreCheckPermission()) {
1721       return;
1722     }
1723 
1724     Path hbaseDir = FSUtils.getRootDir(getConf());
1725     FileSystem fs = hbaseDir.getFileSystem(getConf());
1726     UserProvider userProvider = UserProvider.instantiate(getConf());
1727     UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1728     FileStatus[] files = fs.listStatus(hbaseDir);
1729     for (FileStatus file : files) {
1730       try {
1731         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1732       } catch (AccessDeniedException ace) {
1733         LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1734         errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1735           + " does not have write perms to " + file.getPath()
1736           + ". Please rerun hbck as hdfs user " + file.getOwner());
1737         throw ace;
1738       }
1739     }
1740   }
1741 
1742   /**
1743    * Deletes region from meta table
1744    */
1745   private void deleteMetaRegion(HbckInfo hi) throws IOException {
1746     deleteMetaRegion(hi.metaEntry.getRegionName());
1747   }
1748 
1749   /**
1750    * Deletes region from meta table
1751    */
1752   private void deleteMetaRegion(byte[] metaKey) throws IOException {
1753     Delete d = new Delete(metaKey);
1754     meta.delete(d);
1755     LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1756   }
1757 
1758   /**
1759    * Reset the split parent region info in meta table
1760    */
1761   private void resetSplitParent(HbckInfo hi) throws IOException {
1762     RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
1763     Delete d = new Delete(hi.metaEntry.getRegionName());
1764     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1765     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1766     mutations.add(d);
1767 
1768     HRegionInfo hri = new HRegionInfo(hi.metaEntry);
1769     hri.setOffline(false);
1770     hri.setSplit(false);
1771     Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1772     mutations.add(p);
1773 
1774     meta.mutateRow(mutations);
1775     LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
1776   }
1777 
1778   /**
1779    * This backwards-compatibility wrapper for permanently offlining a region
1780    * that should not be alive.  If the region server does not support the
1781    * "offline" method, it will use the closest unassign method instead.  This
1782    * will basically work until one attempts to disable or delete the affected
1783    * table.  The problem has to do with in-memory only master state, so
1784    * restarting the HMaster or failing over to another should fix this.
1785    */
1786   private void offline(byte[] regionName) throws IOException {
1787     String regionString = Bytes.toStringBinary(regionName);
1788     if (!rsSupportsOffline) {
1789       LOG.warn("Using unassign region " + regionString
1790           + " instead of using offline method, you should"
1791           + " restart HMaster after these repairs");
1792       admin.unassign(regionName, true);
1793       return;
1794     }
1795 
1796     // first time we assume the rs's supports #offline.
1797     try {
1798       LOG.info("Offlining region " + regionString);
1799       admin.offline(regionName);
1800     } catch (IOException ioe) {
1801       String notFoundMsg = "java.lang.NoSuchMethodException: " +
1802         "org.apache.hadoop.hbase.master.HMaster.offline([B)";
1803       if (ioe.getMessage().contains(notFoundMsg)) {
1804         LOG.warn("Using unassign region " + regionString
1805             + " instead of using offline method, you should"
1806             + " restart HMaster after these repairs");
1807         rsSupportsOffline = false; // in the future just use unassign
1808         admin.unassign(regionName, true);
1809         return;
1810       }
1811       throw ioe;
1812     }
1813   }
1814 
1815   private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
1816     undeployRegionsForHbi(hi);
1817     // undeploy replicas of the region (but only if the method is invoked for the primary)
1818     if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1819       return;
1820     }
1821     int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
1822     for (int i = 1; i < numReplicas; i++) {
1823       if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
1824       HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
1825           hi.getPrimaryHRIForDeployedReplica(), i);
1826       HbckInfo h = regionInfoMap.get(hri.getEncodedName());
1827       if (h != null) {
1828         undeployRegionsForHbi(h);
1829         //set skip checks; we undeployed it, and we don't want to evaluate this anymore
1830         //in consistency checks
1831         h.setSkipChecks(true);
1832       }
1833     }
1834   }
1835 
1836   private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
1837     for (OnlineEntry rse : hi.deployedEntries) {
1838       LOG.debug("Undeploy region "  + rse.hri + " from " + rse.hsa);
1839       try {
1840         HBaseFsckRepair.closeRegionSilentlyAndWait(connection, rse.hsa, rse.hri);
1841         offline(rse.hri.getRegionName());
1842       } catch (IOException ioe) {
1843         LOG.warn("Got exception when attempting to offline region "
1844             + Bytes.toString(rse.hri.getRegionName()), ioe);
1845       }
1846     }
1847   }
1848 
1849   /**
1850    * Attempts to undeploy a region from a region server based in information in
1851    * META.  Any operations that modify the file system should make sure that
1852    * its corresponding region is not deployed to prevent data races.
1853    *
1854    * A separate call is required to update the master in-memory region state
1855    * kept in the AssignementManager.  Because disable uses this state instead of
1856    * that found in META, we can't seem to cleanly disable/delete tables that
1857    * have been hbck fixed.  When used on a version of HBase that does not have
1858    * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master
1859    * restart or failover may be required.
1860    */
1861   private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
1862     if (hi.metaEntry == null && hi.hdfsEntry == null) {
1863       undeployRegions(hi);
1864       return;
1865     }
1866 
1867     // get assignment info and hregioninfo from meta.
1868     Get get = new Get(hi.getRegionName());
1869     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1870     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
1871     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
1872     // also get the locations of the replicas to close if the primary region is being closed
1873     if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1874       int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
1875       for (int i = 0; i < numReplicas; i++) {
1876         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
1877         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
1878       }
1879     }
1880     Result r = meta.get(get);
1881     RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
1882     if (rl == null) {
1883       LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
1884           " since meta does not have handle to reach it");
1885       return;
1886     }
1887     for (HRegionLocation h : rl.getRegionLocations()) {
1888       ServerName serverName = h.getServerName();
1889       if (serverName == null) {
1890         errors.reportError("Unable to close region "
1891             + hi.getRegionNameAsString() +  " because meta does not "
1892             + "have handle to reach it.");
1893         continue;
1894       }
1895       HRegionInfo hri = h.getRegionInfo();
1896       if (hri == null) {
1897         LOG.warn("Unable to close region " + hi.getRegionNameAsString()
1898             + " because hbase:meta had invalid or missing "
1899             + HConstants.CATALOG_FAMILY_STR + ":"
1900             + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
1901             + " qualifier value.");
1902         continue;
1903       }
1904       // close the region -- close files and remove assignment
1905       HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
1906     }
1907   }
1908 
1909   private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
1910     KeeperException, InterruptedException {
1911     // If we are trying to fix the errors
1912     if (shouldFixAssignments()) {
1913       errors.print(msg);
1914       undeployRegions(hbi);
1915       setShouldRerun();
1916       HRegionInfo hri = hbi.getHdfsHRI();
1917       if (hri == null) {
1918         hri = hbi.metaEntry;
1919       }
1920       HBaseFsckRepair.fixUnassigned(admin, hri);
1921       HBaseFsckRepair.waitUntilAssigned(admin, hri);
1922 
1923       // also assign replicas if needed (do it only when this call operates on a primary replica)
1924       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
1925       int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
1926       for (int i = 1; i < replicationCount; i++) {
1927         hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
1928         HbckInfo h = regionInfoMap.get(hri.getEncodedName());
1929         if (h != null) {
1930           undeployRegions(h);
1931           //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore
1932           //in consistency checks
1933           h.setSkipChecks(true);
1934         }
1935         HBaseFsckRepair.fixUnassigned(admin, hri);
1936         HBaseFsckRepair.waitUntilAssigned(admin, hri);
1937       }
1938 
1939     }
1940   }
1941 
1942   /**
1943    * Check a single region for consistency and correct deployment.
1944    */
1945   private void checkRegionConsistency(final String key, final HbckInfo hbi)
1946   throws IOException, KeeperException, InterruptedException {
1947 
1948     if (hbi.isSkipChecks()) return;
1949     String descriptiveName = hbi.toString();
1950     boolean inMeta = hbi.metaEntry != null;
1951     // In case not checking HDFS, assume the region is on HDFS
1952     boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
1953     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
1954     boolean isDeployed = !hbi.deployedOn.isEmpty();
1955     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
1956     boolean deploymentMatchesMeta =
1957       hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
1958       hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
1959     boolean splitParent =
1960       (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
1961     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
1962     boolean recentlyModified = inHdfs &&
1963       hbi.getModTime() + timelag > System.currentTimeMillis();
1964 
1965     // ========== First the healthy cases =============
1966     if (hbi.containsOnlyHdfsEdits()) {
1967       return;
1968     }
1969     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
1970       return;
1971     } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
1972       LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
1973         "tabled that is not deployed");
1974       return;
1975     } else if (recentlyModified) {
1976       LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
1977       return;
1978     }
1979     // ========== Cases where the region is not in hbase:meta =============
1980     else if (!inMeta && !inHdfs && !isDeployed) {
1981       // We shouldn't have record of this region at all then!
1982       assert false : "Entry for region with no data";
1983     } else if (!inMeta && !inHdfs && isDeployed) {
1984       errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
1985           + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
1986           "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1987       if (shouldFixAssignments()) {
1988         undeployRegions(hbi);
1989       }
1990 
1991     } else if (!inMeta && inHdfs && !isDeployed) {
1992       if (hbi.isMerged()) {
1993         // This region has already been merged, the remaining hdfs file will be
1994         // cleaned by CatalogJanitor later
1995         hbi.setSkipChecks(true);
1996         LOG.info("Region " + descriptiveName
1997             + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
1998         return;
1999       }
2000       errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
2001           + descriptiveName + " on HDFS, but not listed in hbase:meta " +
2002           "or deployed on any region server");
2003       // restore region consistency of an adopted orphan
2004       if (shouldFixMeta()) {
2005         if (!hbi.isHdfsRegioninfoPresent()) {
2006           LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
2007               +  " in table integrity repair phase if -fixHdfsOrphans was" +
2008               " used.");
2009           return;
2010         }
2011 
2012         HRegionInfo hri = hbi.getHdfsHRI();
2013         TableInfo tableInfo = tablesInfo.get(hri.getTable());
2014         if (tableInfo.regionsFromMeta.isEmpty()) {
2015           for (HbckInfo h : regionInfoMap.values()) {
2016             if (h.getTableName().equals(hri.getTable())) {
2017               if (h.metaEntry != null) tableInfo.regionsFromMeta
2018                   .add((HRegionInfo) h.metaEntry);
2019             }
2020           }
2021           Collections.sort(tableInfo.regionsFromMeta);
2022         }
2023         for (HRegionInfo region : tableInfo.regionsFromMeta) {
2024           if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0
2025               && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(),
2026                 hri.getEndKey()) >= 0)
2027               && Bytes.compareTo(region.getStartKey(), hri.getEndKey()) <= 0) {
2028             if(region.isSplit() || region.isOffline()) continue;
2029             Path regionDir = hbi.getHdfsRegionDir();
2030             FileSystem fs = regionDir.getFileSystem(getConf());
2031             List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDir);
2032             for (Path familyDir : familyDirs) {
2033               List<Path> referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);
2034               for (Path referenceFilePath : referenceFilePaths) {
2035                 Path parentRegionDir =
2036                     StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();
2037                 if (parentRegionDir.toString().endsWith(region.getEncodedName())) {
2038                   LOG.warn(hri + " start and stop keys are in the range of " + region
2039                       + ". The region might not be cleaned up from hdfs when region " + region
2040                       + " split failed. Hence deleting from hdfs.");
2041                   HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs,
2042                     regionDir.getParent(), hri);
2043                   return;
2044                 }
2045               }
2046             }
2047           }
2048         }
2049         LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
2050         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2051         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2052             admin.getClusterStatus().getServers(), numReplicas);
2053 
2054         tryAssignmentRepair(hbi, "Trying to reassign region...");
2055       }
2056 
2057     } else if (!inMeta && inHdfs && isDeployed) {
2058       errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
2059           + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2060       debugLsr(hbi.getHdfsRegionDir());
2061       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
2062         // for replicas, this means that we should undeploy the region (we would have
2063         // gone over the primaries and fixed meta holes in first phase under
2064         // checkAndFixConsistency; we shouldn't get the condition !inMeta at
2065         // this stage unless unwanted replica)
2066         if (shouldFixAssignments()) {
2067           undeployRegionsForHbi(hbi);
2068         }
2069       }
2070       if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2071         if (!hbi.isHdfsRegioninfoPresent()) {
2072           LOG.error("This should have been repaired in table integrity repair phase");
2073           return;
2074         }
2075 
2076         LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
2077         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
2078         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
2079             admin.getClusterStatus().getServers(), numReplicas);
2080         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2081       }
2082 
2083     // ========== Cases where the region is in hbase:meta =============
2084     } else if (inMeta && inHdfs && !isDeployed && splitParent) {
2085       // check whether this is an actual error, or just transient state where parent
2086       // is not cleaned
2087       if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
2088         // check that split daughters are there
2089         HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
2090         HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
2091         if (infoA != null && infoB != null) {
2092           // we already processed or will process daughters. Move on, nothing to see here.
2093           hbi.setSkipChecks(true);
2094           return;
2095         }
2096       }
2097       errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2098           + descriptiveName + " is a split parent in META, in HDFS, "
2099           + "and not deployed on any region server. This could be transient.");
2100       if (shouldFixSplitParents()) {
2101         setShouldRerun();
2102         resetSplitParent(hbi);
2103       }
2104     } else if (inMeta && !inHdfs && !isDeployed) {
2105       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2106           + descriptiveName + " found in META, but not in HDFS "
2107           + "or deployed on any region server.");
2108       if (shouldFixMeta()) {
2109         deleteMetaRegion(hbi);
2110       }
2111     } else if (inMeta && !inHdfs && isDeployed) {
2112       errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2113           + " found in META, but not in HDFS, " +
2114           "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2115       // We treat HDFS as ground truth.  Any information in meta is transient
2116       // and equivalent data can be regenerated.  So, lets unassign and remove
2117       // these problems from META.
2118       if (shouldFixAssignments()) {
2119         errors.print("Trying to fix unassigned region...");
2120         undeployRegions(hbi);
2121       }
2122       if (shouldFixMeta()) {
2123         // wait for it to complete
2124         deleteMetaRegion(hbi);
2125       }
2126     } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2127       errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2128           + " not deployed on any region server.");
2129       tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2130     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2131       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2132           "Region " + descriptiveName + " should not be deployed according " +
2133           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2134       if (shouldFixAssignments()) {
2135         errors.print("Trying to close the region " + descriptiveName);
2136         setShouldRerun();
2137         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2138       }
2139     } else if (inMeta && inHdfs && isMultiplyDeployed) {
2140       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2141           + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
2142           + " but is multiply assigned to region servers " +
2143           Joiner.on(", ").join(hbi.deployedOn));
2144       // If we are trying to fix the errors
2145       if (shouldFixAssignments()) {
2146         errors.print("Trying to fix assignment error...");
2147         setShouldRerun();
2148         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2149       }
2150     } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2151       errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2152           + descriptiveName + " listed in hbase:meta on region server " +
2153           hbi.metaEntry.regionServer + " but found on region server " +
2154           hbi.deployedOn.get(0));
2155       // If we are trying to fix the errors
2156       if (shouldFixAssignments()) {
2157         errors.print("Trying to fix assignment error...");
2158         setShouldRerun();
2159         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2160         HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2161       }
2162     } else {
2163       errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2164           " is in an unforeseen state:" +
2165           " inMeta=" + inMeta +
2166           " inHdfs=" + inHdfs +
2167           " isDeployed=" + isDeployed +
2168           " isMultiplyDeployed=" + isMultiplyDeployed +
2169           " deploymentMatchesMeta=" + deploymentMatchesMeta +
2170           " shouldBeDeployed=" + shouldBeDeployed);
2171     }
2172   }
2173 
2174   /**
2175    * Checks tables integrity. Goes over all regions and scans the tables.
2176    * Collects all the pieces for each table and checks if there are missing,
2177    * repeated or overlapping ones.
2178    * @throws IOException
2179    */
2180   SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2181     tablesInfo = new TreeMap<TableName,TableInfo> ();
2182     LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2183     for (HbckInfo hbi : regionInfoMap.values()) {
2184       // Check only valid, working regions
2185       if (hbi.metaEntry == null) {
2186         // this assumes that consistency check has run loadMetaEntry
2187         Path p = hbi.getHdfsRegionDir();
2188         if (p == null) {
2189           errors.report("No regioninfo in Meta or HDFS. " + hbi);
2190         }
2191 
2192         // TODO test.
2193         continue;
2194       }
2195       if (hbi.metaEntry.regionServer == null) {
2196         errors.detail("Skipping region because no region server: " + hbi);
2197         continue;
2198       }
2199       if (hbi.metaEntry.isOffline()) {
2200         errors.detail("Skipping region because it is offline: " + hbi);
2201         continue;
2202       }
2203       if (hbi.containsOnlyHdfsEdits()) {
2204         errors.detail("Skipping region because it only contains edits" + hbi);
2205         continue;
2206       }
2207 
2208       // Missing regionDir or over-deployment is checked elsewhere. Include
2209       // these cases in modTInfo, so we can evaluate those regions as part of
2210       // the region chain in META
2211       //if (hbi.foundRegionDir == null) continue;
2212       //if (hbi.deployedOn.size() != 1) continue;
2213       if (hbi.deployedOn.size() == 0) continue;
2214 
2215       // We should be safe here
2216       TableName tableName = hbi.metaEntry.getTable();
2217       TableInfo modTInfo = tablesInfo.get(tableName);
2218       if (modTInfo == null) {
2219         modTInfo = new TableInfo(tableName);
2220       }
2221       for (ServerName server : hbi.deployedOn) {
2222         modTInfo.addServer(server);
2223       }
2224 
2225       if (!hbi.isSkipChecks()) {
2226         modTInfo.addRegionInfo(hbi);
2227       }
2228 
2229       tablesInfo.put(tableName, modTInfo);
2230     }
2231 
2232     loadTableInfosForTablesWithNoRegion();
2233 
2234     for (TableInfo tInfo : tablesInfo.values()) {
2235       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2236       if (!tInfo.checkRegionChain(handler)) {
2237         errors.report("Found inconsistency in table " + tInfo.getName());
2238       }
2239     }
2240     return tablesInfo;
2241   }
2242 
2243   /** Loads table info's for tables that may not have been included, since there are no
2244    * regions reported for the table, but table dir is there in hdfs
2245    */
2246   private void loadTableInfosForTablesWithNoRegion() throws IOException {
2247     Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2248     for (HTableDescriptor htd : allTables.values()) {
2249       if (checkMetaOnly && !htd.isMetaTable()) {
2250         continue;
2251       }
2252 
2253       TableName tableName = htd.getTableName();
2254       if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2255         TableInfo tableInfo = new TableInfo(tableName);
2256         tableInfo.htds.add(htd);
2257         tablesInfo.put(htd.getTableName(), tableInfo);
2258       }
2259     }
2260   }
2261 
2262   /**
2263    * Merge hdfs data by moving from contained HbckInfo into targetRegionDir.
2264    * @return number of file move fixes done to merge regions.
2265    */
2266   public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2267     int fileMoves = 0;
2268     String thread = Thread.currentThread().getName();
2269     LOG.debug("[" + thread + "] Contained region dir after close and pause");
2270     debugLsr(contained.getHdfsRegionDir());
2271 
2272     // rename the contained into the container.
2273     FileSystem fs = targetRegionDir.getFileSystem(getConf());
2274     FileStatus[] dirs = null;
2275     try {
2276       dirs = fs.listStatus(contained.getHdfsRegionDir());
2277     } catch (FileNotFoundException fnfe) {
2278       // region we are attempting to merge in is not present!  Since this is a merge, there is
2279       // no harm skipping this region if it does not exist.
2280       if (!fs.exists(contained.getHdfsRegionDir())) {
2281         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2282             + " is missing. Assuming already sidelined or moved.");
2283       } else {
2284         sidelineRegionDir(fs, contained);
2285       }
2286       return fileMoves;
2287     }
2288 
2289     if (dirs == null) {
2290       if (!fs.exists(contained.getHdfsRegionDir())) {
2291         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2292             + " already sidelined.");
2293       } else {
2294         sidelineRegionDir(fs, contained);
2295       }
2296       return fileMoves;
2297     }
2298 
2299     for (FileStatus cf : dirs) {
2300       Path src = cf.getPath();
2301       Path dst =  new Path(targetRegionDir, src.getName());
2302 
2303       if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2304         // do not copy the old .regioninfo file.
2305         continue;
2306       }
2307 
2308       if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2309         // do not copy the .oldlogs files
2310         continue;
2311       }
2312 
2313       LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2314       // FileSystem.rename is inconsistent with directories -- if the
2315       // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2316       // it moves the src into the dst dir resulting in (foo/a/b).  If
2317       // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2318       for (FileStatus hfile : fs.listStatus(src)) {
2319         boolean success = fs.rename(hfile.getPath(), dst);
2320         if (success) {
2321           fileMoves++;
2322         }
2323       }
2324       LOG.debug("[" + thread + "] Sideline directory contents:");
2325       debugLsr(targetRegionDir);
2326     }
2327 
2328     // if all success.
2329     sidelineRegionDir(fs, contained);
2330     LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2331         getSidelineDir());
2332     debugLsr(contained.getHdfsRegionDir());
2333 
2334     return fileMoves;
2335   }
2336 
2337 
2338   static class WorkItemOverlapMerge implements Callable<Void> {
2339     private TableIntegrityErrorHandler handler;
2340     Collection<HbckInfo> overlapgroup;
2341 
2342     WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2343       this.handler = handler;
2344       this.overlapgroup = overlapgroup;
2345     }
2346 
2347     @Override
2348     public Void call() throws Exception {
2349       handler.handleOverlapGroup(overlapgroup);
2350       return null;
2351     }
2352   };
2353 
2354 
2355   /**
2356    * Maintain information about a particular table.
2357    */
2358   public class TableInfo {
2359     TableName tableName;
2360     TreeSet <ServerName> deployedOn;
2361 
2362     // backwards regions
2363     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2364 
2365     // sidelined big overlapped regions
2366     final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2367 
2368     // region split calculator
2369     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2370 
2371     // Histogram of different HTableDescriptors found.  Ideally there is only one!
2372     final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2373 
2374     // key = start split, values = set of splits in problem group
2375     final Multimap<byte[], HbckInfo> overlapGroups =
2376       TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2377 
2378     // list of regions derived from meta entries.
2379     final List<HRegionInfo> regionsFromMeta = new ArrayList<HRegionInfo>();
2380 
2381     TableInfo(TableName name) {
2382       this.tableName = name;
2383       deployedOn = new TreeSet <ServerName>();
2384     }
2385 
2386     /**
2387      * @return descriptor common to all regions.  null if are none or multiple!
2388      */
2389     private HTableDescriptor getHTD() {
2390       if (htds.size() == 1) {
2391         return (HTableDescriptor)htds.toArray()[0];
2392       } else {
2393         LOG.error("None/Multiple table descriptors found for table '"
2394           + tableName + "' regions: " + htds);
2395       }
2396       return null;
2397     }
2398 
2399     public void addRegionInfo(HbckInfo hir) {
2400       if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2401         // end key is absolute end key, just add it.
2402         // ignore replicas other than primary for these checks
2403         if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2404         return;
2405       }
2406 
2407       // if not the absolute end key, check for cycle
2408       if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2409         errors.reportError(
2410             ERROR_CODE.REGION_CYCLE,
2411             String.format("The endkey for this region comes before the "
2412                 + "startkey, startkey=%s, endkey=%s",
2413                 Bytes.toStringBinary(hir.getStartKey()),
2414                 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2415         backwards.add(hir);
2416         return;
2417       }
2418 
2419       // main case, add to split calculator
2420       // ignore replicas other than primary for these checks
2421       if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2422     }
2423 
2424     public void addServer(ServerName server) {
2425       this.deployedOn.add(server);
2426     }
2427 
2428     public TableName getName() {
2429       return tableName;
2430     }
2431 
2432     public int getNumRegions() {
2433       return sc.getStarts().size() + backwards.size();
2434     }
2435 
2436     private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2437       ErrorReporter errors;
2438 
2439       IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2440         this.errors = errors;
2441         setTableInfo(ti);
2442       }
2443 
2444       @Override
2445       public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2446         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2447             "First region should start with an empty key.  You need to "
2448             + " create a new region and regioninfo in HDFS to plug the hole.",
2449             getTableInfo(), hi);
2450       }
2451 
2452       @Override
2453       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2454         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2455             "Last region should end with an empty key. You need to "
2456                 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2457       }
2458 
2459       @Override
2460       public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2461         errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2462             "Region has the same start and end key.", getTableInfo(), hi);
2463       }
2464 
2465       @Override
2466       public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2467         byte[] key = r1.getStartKey();
2468         // dup start key
2469         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2470             "Multiple regions have the same startkey: "
2471             + Bytes.toStringBinary(key), getTableInfo(), r1);
2472         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2473             "Multiple regions have the same startkey: "
2474             + Bytes.toStringBinary(key), getTableInfo(), r2);
2475       }
2476 
2477       @Override
2478       public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2479         errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2480             "There is an overlap in the region chain.",
2481             getTableInfo(), hi1, hi2);
2482       }
2483 
2484       @Override
2485       public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2486         errors.reportError(
2487             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2488             "There is a hole in the region chain between "
2489                 + Bytes.toStringBinary(holeStart) + " and "
2490                 + Bytes.toStringBinary(holeStop)
2491                 + ".  You need to create a new .regioninfo and region "
2492                 + "dir in hdfs to plug the hole.");
2493       }
2494     };
2495 
2496     /**
2497      * This handler fixes integrity errors from hdfs information.  There are
2498      * basically three classes of integrity problems 1) holes, 2) overlaps, and
2499      * 3) invalid regions.
2500      *
2501      * This class overrides methods that fix holes and the overlap group case.
2502      * Individual cases of particular overlaps are handled by the general
2503      * overlap group merge repair case.
2504      *
2505      * If hbase is online, this forces regions offline before doing merge
2506      * operations.
2507      */
2508     private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2509       Configuration conf;
2510 
2511       boolean fixOverlaps = true;
2512 
2513       HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2514           boolean fixHoles, boolean fixOverlaps) {
2515         super(ti, errors);
2516         this.conf = conf;
2517         this.fixOverlaps = fixOverlaps;
2518         // TODO properly use fixHoles
2519       }
2520 
2521       /**
2522        * This is a special case hole -- when the first region of a table is
2523        * missing from META, HBase doesn't acknowledge the existance of the
2524        * table.
2525        */
2526       @Override
2527       public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2528         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2529             "First region should start with an empty key.  Creating a new " +
2530             "region and regioninfo in HDFS to plug the hole.",
2531             getTableInfo(), next);
2532         HTableDescriptor htd = getTableInfo().getHTD();
2533         // from special EMPTY_START_ROW to next region's startKey
2534         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2535             HConstants.EMPTY_START_ROW, next.getStartKey());
2536 
2537         // TODO test
2538         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2539         LOG.info("Table region start key was not empty.  Created new empty region: "
2540             + newRegion + " " +region);
2541         fixes++;
2542       }
2543 
2544       @Override
2545       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2546         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2547             "Last region should end with an empty key.  Creating a new "
2548                 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2549         HTableDescriptor htd = getTableInfo().getHTD();
2550         // from curEndKey to EMPTY_START_ROW
2551         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2552             HConstants.EMPTY_START_ROW);
2553 
2554         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2555         LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
2556             + " " + region);
2557         fixes++;
2558       }
2559 
2560       /**
2561        * There is a hole in the hdfs regions that violates the table integrity
2562        * rules.  Create a new empty region that patches the hole.
2563        */
2564       @Override
2565       public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2566         errors.reportError(
2567             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2568             "There is a hole in the region chain between "
2569                 + Bytes.toStringBinary(holeStartKey) + " and "
2570                 + Bytes.toStringBinary(holeStopKey)
2571                 + ".  Creating a new regioninfo and region "
2572                 + "dir in hdfs to plug the hole.");
2573         HTableDescriptor htd = getTableInfo().getHTD();
2574         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2575         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2576         LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2577         fixes++;
2578       }
2579 
2580       /**
2581        * This takes set of overlapping regions and merges them into a single
2582        * region.  This covers cases like degenerate regions, shared start key,
2583        * general overlaps, duplicate ranges, and partial overlapping regions.
2584        *
2585        * Cases:
2586        * - Clean regions that overlap
2587        * - Only .oldlogs regions (can't find start/stop range, or figure out)
2588        *
2589        * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
2590        */
2591       @Override
2592       public void handleOverlapGroup(Collection<HbckInfo> overlap)
2593           throws IOException {
2594         Preconditions.checkNotNull(overlap);
2595         Preconditions.checkArgument(overlap.size() >0);
2596 
2597         if (!this.fixOverlaps) {
2598           LOG.warn("Not attempting to repair overlaps.");
2599           return;
2600         }
2601 
2602         if (overlap.size() > maxMerge) {
2603           LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2604             "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2605           if (sidelineBigOverlaps) {
2606             // we only sideline big overlapped groups that exceeds the max number of regions to merge
2607             sidelineBigOverlaps(overlap);
2608           }
2609           return;
2610         }
2611 
2612         mergeOverlaps(overlap);
2613       }
2614 
2615       void mergeOverlaps(Collection<HbckInfo> overlap)
2616           throws IOException {
2617         String thread = Thread.currentThread().getName();
2618         LOG.info("== [" + thread + "] Merging regions into one region: "
2619           + Joiner.on(",").join(overlap));
2620         // get the min / max range and close all concerned regions
2621         Pair<byte[], byte[]> range = null;
2622         for (HbckInfo hi : overlap) {
2623           if (range == null) {
2624             range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2625           } else {
2626             if (RegionSplitCalculator.BYTES_COMPARATOR
2627                 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2628               range.setFirst(hi.getStartKey());
2629             }
2630             if (RegionSplitCalculator.BYTES_COMPARATOR
2631                 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2632               range.setSecond(hi.getEndKey());
2633             }
2634           }
2635           // need to close files so delete can happen.
2636           LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
2637           LOG.debug("[" + thread + "] Contained region dir before close");
2638           debugLsr(hi.getHdfsRegionDir());
2639           try {
2640             LOG.info("[" + thread + "] Closing region: " + hi);
2641             closeRegion(hi);
2642           } catch (IOException ioe) {
2643             LOG.warn("[" + thread + "] Was unable to close region " + hi
2644               + ".  Just continuing... ", ioe);
2645           } catch (InterruptedException e) {
2646             LOG.warn("[" + thread + "] Was unable to close region " + hi
2647               + ".  Just continuing... ", e);
2648           }
2649 
2650           try {
2651             LOG.info("[" + thread + "] Offlining region: " + hi);
2652             offline(hi.getRegionName());
2653           } catch (IOException ioe) {
2654             LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2655               + ".  Just continuing... ", ioe);
2656           }
2657         }
2658 
2659         // create new empty container region.
2660         HTableDescriptor htd = getTableInfo().getHTD();
2661         // from start key to end Key
2662         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
2663             range.getSecond());
2664         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2665         LOG.info("[" + thread + "] Created new empty container region: " +
2666             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
2667         debugLsr(region.getRegionFileSystem().getRegionDir());
2668 
2669         // all target regions are closed, should be able to safely cleanup.
2670         boolean didFix= false;
2671         Path target = region.getRegionFileSystem().getRegionDir();
2672         for (HbckInfo contained : overlap) {
2673           LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
2674           int merges = mergeRegionDirs(target, contained);
2675           if (merges > 0) {
2676             didFix = true;
2677           }
2678         }
2679         if (didFix) {
2680           fixes++;
2681         }
2682       }
2683 
2684       /**
2685        * Sideline some regions in a big overlap group so that it
2686        * will have fewer regions, and it is easier to merge them later on.
2687        *
2688        * @param bigOverlap the overlapped group with regions more than maxMerge
2689        * @throws IOException
2690        */
2691       void sidelineBigOverlaps(
2692           Collection<HbckInfo> bigOverlap) throws IOException {
2693         int overlapsToSideline = bigOverlap.size() - maxMerge;
2694         if (overlapsToSideline > maxOverlapsToSideline) {
2695           overlapsToSideline = maxOverlapsToSideline;
2696         }
2697         List<HbckInfo> regionsToSideline =
2698           RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
2699         FileSystem fs = FileSystem.get(conf);
2700         for (HbckInfo regionToSideline: regionsToSideline) {
2701           try {
2702             LOG.info("Closing region: " + regionToSideline);
2703             closeRegion(regionToSideline);
2704           } catch (IOException ioe) {
2705             LOG.warn("Was unable to close region " + regionToSideline
2706               + ".  Just continuing... ", ioe);
2707           } catch (InterruptedException e) {
2708             LOG.warn("Was unable to close region " + regionToSideline
2709               + ".  Just continuing... ", e);
2710           }
2711 
2712           try {
2713             LOG.info("Offlining region: " + regionToSideline);
2714             offline(regionToSideline.getRegionName());
2715           } catch (IOException ioe) {
2716             LOG.warn("Unable to offline region from master: " + regionToSideline
2717               + ".  Just continuing... ", ioe);
2718           }
2719 
2720           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
2721           Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
2722           if (sidelineRegionDir != null) {
2723             sidelinedRegions.put(sidelineRegionDir, regionToSideline);
2724             LOG.info("After sidelined big overlapped region: "
2725               + regionToSideline.getRegionNameAsString()
2726               + " to " + sidelineRegionDir.toString());
2727             fixes++;
2728           }
2729         }
2730       }
2731     }
2732 
2733     /**
2734      * Check the region chain (from META) of this table.  We are looking for
2735      * holes, overlaps, and cycles.
2736      * @return false if there are errors
2737      * @throws IOException
2738      */
2739     public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
2740       // When table is disabled no need to check for the region chain. Some of the regions
2741       // accidently if deployed, this below code might report some issues like missing start
2742       // or end regions or region hole in chain and may try to fix which is unwanted.
2743       if (disabledTables.contains(this.tableName)) {
2744         return true;
2745       }
2746       int originalErrorsCount = errors.getErrorList().size();
2747       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
2748       SortedSet<byte[]> splits = sc.getSplits();
2749 
2750       byte[] prevKey = null;
2751       byte[] problemKey = null;
2752 
2753       if (splits.size() == 0) {
2754         // no region for this table
2755         handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
2756       }
2757 
2758       for (byte[] key : splits) {
2759         Collection<HbckInfo> ranges = regions.get(key);
2760         if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
2761           for (HbckInfo rng : ranges) {
2762             handler.handleRegionStartKeyNotEmpty(rng);
2763           }
2764         }
2765 
2766         // check for degenerate ranges
2767         for (HbckInfo rng : ranges) {
2768           // special endkey case converts '' to null
2769           byte[] endKey = rng.getEndKey();
2770           endKey = (endKey.length == 0) ? null : endKey;
2771           if (Bytes.equals(rng.getStartKey(),endKey)) {
2772             handler.handleDegenerateRegion(rng);
2773           }
2774         }
2775 
2776         if (ranges.size() == 1) {
2777           // this split key is ok -- no overlap, not a hole.
2778           if (problemKey != null) {
2779             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2780           }
2781           problemKey = null; // fell through, no more problem.
2782         } else if (ranges.size() > 1) {
2783           // set the new problem key group name, if already have problem key, just
2784           // keep using it.
2785           if (problemKey == null) {
2786             // only for overlap regions.
2787             LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
2788             problemKey = key;
2789           }
2790           overlapGroups.putAll(problemKey, ranges);
2791 
2792           // record errors
2793           ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
2794           //  this dumb and n^2 but this shouldn't happen often
2795           for (HbckInfo r1 : ranges) {
2796             if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2797             subRange.remove(r1);
2798             for (HbckInfo r2 : subRange) {
2799               if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2800               if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
2801                 handler.handleDuplicateStartKeys(r1,r2);
2802               } else {
2803                 // overlap
2804                 handler.handleOverlapInRegionChain(r1, r2);
2805               }
2806             }
2807           }
2808 
2809         } else if (ranges.size() == 0) {
2810           if (problemKey != null) {
2811             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2812           }
2813           problemKey = null;
2814 
2815           byte[] holeStopKey = sc.getSplits().higher(key);
2816           // if higher key is null we reached the top.
2817           if (holeStopKey != null) {
2818             // hole
2819             handler.handleHoleInRegionChain(key, holeStopKey);
2820           }
2821         }
2822         prevKey = key;
2823       }
2824 
2825       // When the last region of a table is proper and having an empty end key, 'prevKey'
2826       // will be null.
2827       if (prevKey != null) {
2828         handler.handleRegionEndKeyNotEmpty(prevKey);
2829       }
2830 
2831       // TODO fold this into the TableIntegrityHandler
2832       if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
2833         LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
2834             " false to run serially.");
2835         boolean ok = handleOverlapsParallel(handler, prevKey);
2836         if (!ok) {
2837           return false;
2838         }
2839       } else {
2840         LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
2841             " true to run in parallel.");
2842         for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2843           handler.handleOverlapGroup(overlap);
2844         }
2845       }
2846 
2847       if (details) {
2848         // do full region split map dump
2849         errors.print("---- Table '"  +  this.tableName
2850             + "': region split map");
2851         dump(splits, regions);
2852         errors.print("---- Table '"  +  this.tableName
2853             + "': overlap groups");
2854         dumpOverlapProblems(overlapGroups);
2855         errors.print("There are " + overlapGroups.keySet().size()
2856             + " overlap groups with " + overlapGroups.size()
2857             + " overlapping regions");
2858       }
2859       if (!sidelinedRegions.isEmpty()) {
2860         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
2861         errors.print("---- Table '"  +  this.tableName
2862             + "': sidelined big overlapped regions");
2863         dumpSidelinedRegions(sidelinedRegions);
2864       }
2865       return errors.getErrorList().size() == originalErrorsCount;
2866     }
2867 
2868     private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
2869         throws IOException {
2870       // we parallelize overlap handler for the case we have lots of groups to fix.  We can
2871       // safely assume each group is independent.
2872       List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
2873       List<Future<Void>> rets;
2874       for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2875         //
2876         merges.add(new WorkItemOverlapMerge(overlap, handler));
2877       }
2878       try {
2879         rets = executor.invokeAll(merges);
2880       } catch (InterruptedException e) {
2881         LOG.error("Overlap merges were interrupted", e);
2882         return false;
2883       }
2884       for(int i=0; i<merges.size(); i++) {
2885         WorkItemOverlapMerge work = merges.get(i);
2886         Future<Void> f = rets.get(i);
2887         try {
2888           f.get();
2889         } catch(ExecutionException e) {
2890           LOG.warn("Failed to merge overlap group" + work, e.getCause());
2891         } catch (InterruptedException e) {
2892           LOG.error("Waiting for overlap merges was interrupted", e);
2893           return false;
2894         }
2895       }
2896       return true;
2897     }
2898 
2899     /**
2900      * This dumps data in a visually reasonable way for visual debugging
2901      *
2902      * @param splits
2903      * @param regions
2904      */
2905     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
2906       // we display this way because the last end key should be displayed as well.
2907       StringBuilder sb = new StringBuilder();
2908       for (byte[] k : splits) {
2909         sb.setLength(0); // clear out existing buffer, if any.
2910         sb.append(Bytes.toStringBinary(k) + ":\t");
2911         for (HbckInfo r : regions.get(k)) {
2912           sb.append("[ "+ r.toString() + ", "
2913               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
2914         }
2915         errors.print(sb.toString());
2916       }
2917     }
2918   }
2919 
2920   public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
2921     // we display this way because the last end key should be displayed as
2922     // well.
2923     for (byte[] k : regions.keySet()) {
2924       errors.print(Bytes.toStringBinary(k) + ":");
2925       for (HbckInfo r : regions.get(k)) {
2926         errors.print("[ " + r.toString() + ", "
2927             + Bytes.toStringBinary(r.getEndKey()) + "]");
2928       }
2929       errors.print("----");
2930     }
2931   }
2932 
2933   public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
2934     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
2935       TableName tableName = entry.getValue().getTableName();
2936       Path path = entry.getKey();
2937       errors.print("This sidelined region dir should be bulk loaded: "
2938         + path.toString());
2939       errors.print("Bulk load command looks like: "
2940         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
2941         + path.toUri().getPath() + " "+ tableName);
2942     }
2943   }
2944 
2945   public Multimap<byte[], HbckInfo> getOverlapGroups(
2946       TableName table) {
2947     TableInfo ti = tablesInfo.get(table);
2948     return ti.overlapGroups;
2949   }
2950 
2951   /**
2952    * Return a list of user-space table names whose metadata have not been
2953    * modified in the last few milliseconds specified by timelag
2954    * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
2955    * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
2956    * milliseconds specified by timelag, then the table is a candidate to be returned.
2957    * @return tables that have not been modified recently
2958    * @throws IOException if an error is encountered
2959    */
2960   HTableDescriptor[] getTables(AtomicInteger numSkipped) {
2961     List<TableName> tableNames = new ArrayList<TableName>();
2962     long now = System.currentTimeMillis();
2963 
2964     for (HbckInfo hbi : regionInfoMap.values()) {
2965       MetaEntry info = hbi.metaEntry;
2966 
2967       // if the start key is zero, then we have found the first region of a table.
2968       // pick only those tables that were not modified in the last few milliseconds.
2969       if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
2970         if (info.modTime + timelag < now) {
2971           tableNames.add(info.getTable());
2972         } else {
2973           numSkipped.incrementAndGet(); // one more in-flux table
2974         }
2975       }
2976     }
2977     return getHTableDescriptors(tableNames);
2978   }
2979 
2980   HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
2981     HTableDescriptor[] htd = new HTableDescriptor[0];
2982     Admin admin = null;
2983     try {
2984       LOG.info("getHTableDescriptors == tableNames => " + tableNames);
2985       admin = new HBaseAdmin(getConf());
2986       htd = admin.getTableDescriptorsByTableName(tableNames);
2987     } catch (IOException e) {
2988       LOG.debug("Exception getting table descriptors", e);
2989     } finally {
2990       if (admin != null) {
2991         try {
2992           admin.close();
2993         } catch (IOException e) {
2994           LOG.debug("Exception closing HBaseAdmin", e);
2995         }
2996       }
2997     }
2998     return htd;
2999   }
3000 
3001   /**
3002    * Gets the entry in regionInfo corresponding to the the given encoded
3003    * region name. If the region has not been seen yet, a new entry is added
3004    * and returned.
3005    */
3006   private synchronized HbckInfo getOrCreateInfo(String name) {
3007     HbckInfo hbi = regionInfoMap.get(name);
3008     if (hbi == null) {
3009       hbi = new HbckInfo(null);
3010       regionInfoMap.put(name, hbi);
3011     }
3012     return hbi;
3013   }
3014 
3015   private void checkAndFixTableLocks() throws IOException {
3016     TableLockChecker checker = new TableLockChecker(createZooKeeperWatcher(), errors);
3017     checker.checkTableLocks();
3018 
3019     if (this.fixTableLocks) {
3020       checker.fixExpiredTableLocks();
3021     }
3022   }
3023 
3024   /**
3025     * Check values in regionInfo for hbase:meta
3026     * Check if zero or more than one regions with hbase:meta are found.
3027     * If there are inconsistencies (i.e. zero or more than one regions
3028     * pretend to be holding the hbase:meta) try to fix that and report an error.
3029     * @throws IOException from HBaseFsckRepair functions
3030    * @throws KeeperException
3031    * @throws InterruptedException
3032     */
3033   boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
3034     List<HbckInfo> metaRegions = Lists.newArrayList();
3035     for (HbckInfo value : regionInfoMap.values()) {
3036       if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
3037         metaRegions.add(value);
3038       }
3039     }
3040 
3041     // There will be always one entry in regionInfoMap corresponding to hbase:meta
3042     // Check the deployed servers. It should be exactly one server.
3043     List<ServerName> servers = new ArrayList<ServerName>();
3044     HbckInfo metaHbckInfo = null;
3045     if (!metaRegions.isEmpty()) {
3046       metaHbckInfo = metaRegions.get(0);
3047       servers = metaHbckInfo.deployedOn;
3048     }
3049     if (servers.size() != 1) {
3050       if (servers.size() == 0) {
3051         errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta is not found on any region.");
3052         if (shouldFixAssignments()) {
3053           errors.print("Trying to fix a problem with hbase:meta..");
3054           setShouldRerun();
3055           // try to fix it (treat it as unassigned region)
3056           HBaseFsckRepair.fixUnassigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
3057           HBaseFsckRepair.waitUntilAssigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
3058         }
3059       } else if (servers.size() > 1) {
3060         errors
3061             .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta is found on more than one region.");
3062         if (shouldFixAssignments()) {
3063           if (metaHbckInfo == null) {
3064             errors.print(
3065               "Unable to fix problem with hbase:meta due to hbase:meta region info missing");
3066             return false;
3067           }
3068           errors.print("Trying to fix a problem with hbase:meta..");
3069           setShouldRerun();
3070           // try fix it (treat is a dupe assignment)
3071           HBaseFsckRepair.fixMultiAssignment(connection, metaHbckInfo.metaEntry, servers);
3072         }
3073       }
3074       // rerun hbck with hopefully fixed META
3075       return false;
3076     }
3077     // no errors, so continue normally
3078     return true;
3079   }
3080 
3081   /**
3082    * Scan hbase:meta, adding all regions found to the regionInfo map.
3083    * @throws IOException if an error is encountered
3084    */
3085   boolean loadMetaEntries() throws IOException {
3086     MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
3087       int countRecord = 1;
3088 
3089       // comparator to sort KeyValues with latest modtime
3090       final Comparator<Cell> comp = new Comparator<Cell>() {
3091         @Override
3092         public int compare(Cell k1, Cell k2) {
3093           return (int)(k1.getTimestamp() - k2.getTimestamp());
3094         }
3095       };
3096 
3097       @Override
3098       public boolean processRow(Result result) throws IOException {
3099         try {
3100 
3101           // record the latest modification of this META record
3102           long ts =  Collections.max(result.listCells(), comp).getTimestamp();
3103           RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
3104           if (rl == null) {
3105             emptyRegionInfoQualifiers.add(result);
3106             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3107               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3108             return true;
3109           }
3110           ServerName sn = null;
3111           if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
3112               rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
3113             emptyRegionInfoQualifiers.add(result);
3114             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3115               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3116             return true;
3117           }
3118           HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
3119           if (!(isTableIncluded(hri.getTable())
3120               || hri.isMetaRegion())) {
3121             return true;
3122           }
3123           PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
3124           for (HRegionLocation h : rl.getRegionLocations()) {
3125             if (h == null || h.getRegionInfo() == null) {
3126               continue;
3127             }
3128             sn = h.getServerName();
3129             hri = h.getRegionInfo();
3130 
3131             MetaEntry m = null;
3132             if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
3133               m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
3134             } else {
3135               m = new MetaEntry(hri, sn, ts, null, null);
3136             }
3137             HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
3138             if (previous == null) {
3139               regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
3140             } else if (previous.metaEntry == null) {
3141               previous.metaEntry = m;
3142             } else {
3143               throw new IOException("Two entries in hbase:meta are same " + previous);
3144             }
3145           }
3146           PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
3147           for (HRegionInfo mergeRegion : new HRegionInfo[] {
3148               mergeRegions.getFirst(), mergeRegions.getSecond() }) {
3149             if (mergeRegion != null) {
3150               // This region is already been merged
3151               HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
3152               hbInfo.setMerged(true);
3153             }
3154           }
3155 
3156           // show proof of progress to the user, once for every 100 records.
3157           if (countRecord % 100 == 0) {
3158             errors.progress();
3159           }
3160           countRecord++;
3161           return true;
3162         } catch (RuntimeException e) {
3163           LOG.error("Result=" + result);
3164           throw e;
3165         }
3166       }
3167     };
3168     if (!checkMetaOnly) {
3169       // Scan hbase:meta to pick up user regions
3170       MetaScanner.metaScan(connection, visitor);
3171     }
3172 
3173     errors.print("");
3174     return true;
3175   }
3176 
3177   /**
3178    * Stores the regioninfo entries scanned from META
3179    */
3180   static class MetaEntry extends HRegionInfo {
3181     ServerName regionServer;   // server hosting this region
3182     long modTime;          // timestamp of most recent modification metadata
3183     HRegionInfo splitA, splitB; //split daughters
3184 
3185     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3186       this(rinfo, regionServer, modTime, null, null);
3187     }
3188 
3189     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3190         HRegionInfo splitA, HRegionInfo splitB) {
3191       super(rinfo);
3192       this.regionServer = regionServer;
3193       this.modTime = modTime;
3194       this.splitA = splitA;
3195       this.splitB = splitB;
3196     }
3197 
3198     @Override
3199     public boolean equals(Object o) {
3200       boolean superEq = super.equals(o);
3201       if (!superEq) {
3202         return superEq;
3203       }
3204 
3205       MetaEntry me = (MetaEntry) o;
3206       if (!regionServer.equals(me.regionServer)) {
3207         return false;
3208       }
3209       return (modTime == me.modTime);
3210     }
3211 
3212     @Override
3213     public int hashCode() {
3214       int hash = Arrays.hashCode(getRegionName());
3215       hash ^= getRegionId();
3216       hash ^= Arrays.hashCode(getStartKey());
3217       hash ^= Arrays.hashCode(getEndKey());
3218       hash ^= Boolean.valueOf(isOffline()).hashCode();
3219       hash ^= getTable().hashCode();
3220       if (regionServer != null) {
3221         hash ^= regionServer.hashCode();
3222       }
3223       hash ^= modTime;
3224       return hash;
3225     }
3226   }
3227 
3228   /**
3229    * Stores the regioninfo entries from HDFS
3230    */
3231   static class HdfsEntry {
3232     HRegionInfo hri;
3233     Path hdfsRegionDir = null;
3234     long hdfsRegionDirModTime  = 0;
3235     boolean hdfsRegioninfoFilePresent = false;
3236     boolean hdfsOnlyEdits = false;
3237   }
3238 
3239   /**
3240    * Stores the regioninfo retrieved from Online region servers.
3241    */
3242   static class OnlineEntry {
3243     HRegionInfo hri;
3244     ServerName hsa;
3245 
3246     @Override
3247     public String toString() {
3248       return hsa.toString() + ";" + hri.getRegionNameAsString();
3249     }
3250   }
3251 
3252   /**
3253    * Maintain information about a particular region.  It gathers information
3254    * from three places -- HDFS, META, and region servers.
3255    */
3256   public static class HbckInfo implements KeyRange {
3257     private MetaEntry metaEntry = null; // info in META
3258     private HdfsEntry hdfsEntry = null; // info in HDFS
3259     private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
3260     private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
3261     private boolean skipChecks = false; // whether to skip further checks to this region info.
3262     private boolean isMerged = false;// whether this region has already been merged into another one
3263     private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3264     private HRegionInfo primaryHRIForDeployedReplica = null;
3265 
3266     HbckInfo(MetaEntry metaEntry) {
3267       this.metaEntry = metaEntry;
3268     }
3269 
3270     public int getReplicaId() {
3271       if (metaEntry != null) return metaEntry.getReplicaId();
3272       return deployedReplicaId;
3273     }
3274 
3275     public synchronized void addServer(HRegionInfo hri, ServerName server) {
3276       OnlineEntry rse = new OnlineEntry() ;
3277       rse.hri = hri;
3278       rse.hsa = server;
3279       this.deployedEntries.add(rse);
3280       this.deployedOn.add(server);
3281       // save the replicaId that we see deployed in the cluster
3282       this.deployedReplicaId = hri.getReplicaId();
3283       this.primaryHRIForDeployedReplica =
3284           RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3285     }
3286 
3287     @Override
3288     public synchronized String toString() {
3289       StringBuilder sb = new StringBuilder();
3290       sb.append("{ meta => ");
3291       sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3292       sb.append( ", hdfs => " + getHdfsRegionDir());
3293       sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3294       sb.append( ", replicaId => " + getReplicaId());
3295       sb.append(" }");
3296       return sb.toString();
3297     }
3298 
3299     @Override
3300     public byte[] getStartKey() {
3301       if (this.metaEntry != null) {
3302         return this.metaEntry.getStartKey();
3303       } else if (this.hdfsEntry != null) {
3304         return this.hdfsEntry.hri.getStartKey();
3305       } else {
3306         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3307         return null;
3308       }
3309     }
3310 
3311     @Override
3312     public byte[] getEndKey() {
3313       if (this.metaEntry != null) {
3314         return this.metaEntry.getEndKey();
3315       } else if (this.hdfsEntry != null) {
3316         return this.hdfsEntry.hri.getEndKey();
3317       } else {
3318         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3319         return null;
3320       }
3321     }
3322 
3323     public TableName getTableName() {
3324       if (this.metaEntry != null) {
3325         return this.metaEntry.getTable();
3326       } else if (this.hdfsEntry != null) {
3327         // we are only guaranteed to have a path and not an HRI for hdfsEntry,
3328         // so we get the name from the Path
3329         Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3330         return FSUtils.getTableName(tableDir);
3331       } else {
3332         // return the info from the first online/deployed hri
3333         for (OnlineEntry e : deployedEntries) {
3334           return e.hri.getTable();
3335         }
3336         return null;
3337       }
3338     }
3339 
3340     public String getRegionNameAsString() {
3341       if (metaEntry != null) {
3342         return metaEntry.getRegionNameAsString();
3343       } else if (hdfsEntry != null) {
3344         if (hdfsEntry.hri != null) {
3345           return hdfsEntry.hri.getRegionNameAsString();
3346         }
3347       } else {
3348         // return the info from the first online/deployed hri
3349         for (OnlineEntry e : deployedEntries) {
3350           return e.hri.getRegionNameAsString();
3351         }
3352       }
3353       return null;
3354     }
3355 
3356     public byte[] getRegionName() {
3357       if (metaEntry != null) {
3358         return metaEntry.getRegionName();
3359       } else if (hdfsEntry != null) {
3360         return hdfsEntry.hri.getRegionName();
3361       } else {
3362         // return the info from the first online/deployed hri
3363         for (OnlineEntry e : deployedEntries) {
3364           return e.hri.getRegionName();
3365         }
3366         return null;
3367       }
3368     }
3369 
3370     public HRegionInfo getPrimaryHRIForDeployedReplica() {
3371       return primaryHRIForDeployedReplica;
3372     }
3373 
3374     Path getHdfsRegionDir() {
3375       if (hdfsEntry == null) {
3376         return null;
3377       }
3378       return hdfsEntry.hdfsRegionDir;
3379     }
3380 
3381     boolean containsOnlyHdfsEdits() {
3382       if (hdfsEntry == null) {
3383         return false;
3384       }
3385       return hdfsEntry.hdfsOnlyEdits;
3386     }
3387 
3388     boolean isHdfsRegioninfoPresent() {
3389       if (hdfsEntry == null) {
3390         return false;
3391       }
3392       return hdfsEntry.hdfsRegioninfoFilePresent;
3393     }
3394 
3395     long getModTime() {
3396       if (hdfsEntry == null) {
3397         return 0;
3398       }
3399       return hdfsEntry.hdfsRegionDirModTime;
3400     }
3401 
3402     HRegionInfo getHdfsHRI() {
3403       if (hdfsEntry == null) {
3404         return null;
3405       }
3406       return hdfsEntry.hri;
3407     }
3408 
3409     public void setSkipChecks(boolean skipChecks) {
3410       this.skipChecks = skipChecks;
3411     }
3412 
3413     public boolean isSkipChecks() {
3414       return skipChecks;
3415     }
3416 
3417     public void setMerged(boolean isMerged) {
3418       this.isMerged = isMerged;
3419     }
3420 
3421     public boolean isMerged() {
3422       return this.isMerged;
3423     }
3424   }
3425 
3426   final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3427     @Override
3428     public int compare(HbckInfo l, HbckInfo r) {
3429       if (l == r) {
3430         // same instance
3431         return 0;
3432       }
3433 
3434       int tableCompare = l.getTableName().compareTo(r.getTableName());
3435       if (tableCompare != 0) {
3436         return tableCompare;
3437       }
3438 
3439       int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3440           l.getStartKey(), r.getStartKey());
3441       if (startComparison != 0) {
3442         return startComparison;
3443       }
3444 
3445       // Special case for absolute endkey
3446       byte[] endKey = r.getEndKey();
3447       endKey = (endKey.length == 0) ? null : endKey;
3448       byte[] endKey2 = l.getEndKey();
3449       endKey2 = (endKey2.length == 0) ? null : endKey2;
3450       int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3451           endKey2,  endKey);
3452 
3453       if (endComparison != 0) {
3454         return endComparison;
3455       }
3456 
3457       // use regionId as tiebreaker.
3458       // Null is considered after all possible values so make it bigger.
3459       if (l.hdfsEntry == null && r.hdfsEntry == null) {
3460         return 0;
3461       }
3462       if (l.hdfsEntry == null && r.hdfsEntry != null) {
3463         return 1;
3464       }
3465       // l.hdfsEntry must not be null
3466       if (r.hdfsEntry == null) {
3467         return -1;
3468       }
3469       // both l.hdfsEntry and r.hdfsEntry must not be null.
3470       return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3471     }
3472   };
3473 
3474   /**
3475    * Prints summary of all tables found on the system.
3476    */
3477   private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3478     StringBuilder sb = new StringBuilder();
3479     errors.print("Summary:");
3480     for (TableInfo tInfo : tablesInfo.values()) {
3481       if (errors.tableHasErrors(tInfo)) {
3482         errors.print("Table " + tInfo.getName() + " is inconsistent.");
3483       } else {
3484         errors.print("  " + tInfo.getName() + " is okay.");
3485       }
3486       errors.print("    Number of regions: " + tInfo.getNumRegions());
3487       sb.setLength(0); // clear out existing buffer, if any.
3488       sb.append("    Deployed on: ");
3489       for (ServerName server : tInfo.deployedOn) {
3490         sb.append(" " + server.toString());
3491       }
3492       errors.print(sb.toString());
3493     }
3494   }
3495 
3496   static ErrorReporter getErrorReporter(
3497       final Configuration conf) throws ClassNotFoundException {
3498     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3499     return ReflectionUtils.newInstance(reporter, conf);
3500   }
3501 
3502   public interface ErrorReporter {
3503     enum ERROR_CODE {
3504       UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3505       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
3506       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3507       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3508       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3509       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3510       WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR
3511     }
3512     void clear();
3513     void report(String message);
3514     void reportError(String message);
3515     void reportError(ERROR_CODE errorCode, String message);
3516     void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3517     void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3518     void reportError(
3519       ERROR_CODE errorCode,
3520       String message,
3521       TableInfo table,
3522       HbckInfo info1,
3523       HbckInfo info2
3524     );
3525     int summarize();
3526     void detail(String details);
3527     ArrayList<ERROR_CODE> getErrorList();
3528     void progress();
3529     void print(String message);
3530     void resetErrors();
3531     boolean tableHasErrors(TableInfo table);
3532   }
3533 
3534   static class PrintingErrorReporter implements ErrorReporter {
3535     public int errorCount = 0;
3536     private int showProgress;
3537 
3538     Set<TableInfo> errorTables = new HashSet<TableInfo>();
3539 
3540     // for use by unit tests to verify which errors were discovered
3541     private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3542 
3543     @Override
3544     public void clear() {
3545       errorTables.clear();
3546       errorList.clear();
3547       errorCount = 0;
3548     }
3549 
3550     @Override
3551     public synchronized void reportError(ERROR_CODE errorCode, String message) {
3552       if (errorCode == ERROR_CODE.WRONG_USAGE) {
3553         System.err.println(message);
3554         return;
3555       }
3556 
3557       errorList.add(errorCode);
3558       if (!summary) {
3559         System.out.println("ERROR: " + message);
3560       }
3561       errorCount++;
3562       showProgress = 0;
3563     }
3564 
3565     @Override
3566     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
3567       errorTables.add(table);
3568       reportError(errorCode, message);
3569     }
3570 
3571     @Override
3572     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3573                                          HbckInfo info) {
3574       errorTables.add(table);
3575       String reference = "(region " + info.getRegionNameAsString() + ")";
3576       reportError(errorCode, reference + " " + message);
3577     }
3578 
3579     @Override
3580     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3581                                          HbckInfo info1, HbckInfo info2) {
3582       errorTables.add(table);
3583       String reference = "(regions " + info1.getRegionNameAsString()
3584           + " and " + info2.getRegionNameAsString() + ")";
3585       reportError(errorCode, reference + " " + message);
3586     }
3587 
3588     @Override
3589     public synchronized void reportError(String message) {
3590       reportError(ERROR_CODE.UNKNOWN, message);
3591     }
3592 
3593     /**
3594      * Report error information, but do not increment the error count.  Intended for cases
3595      * where the actual error would have been reported previously.
3596      * @param message
3597      */
3598     @Override
3599     public synchronized void report(String message) {
3600       if (! summary) {
3601         System.out.println("ERROR: " + message);
3602       }
3603       showProgress = 0;
3604     }
3605 
3606     @Override
3607     public synchronized int summarize() {
3608       System.out.println(Integer.toString(errorCount) +
3609                          " inconsistencies detected.");
3610       if (errorCount == 0) {
3611         System.out.println("Status: OK");
3612         return 0;
3613       } else {
3614         System.out.println("Status: INCONSISTENT");
3615         return -1;
3616       }
3617     }
3618 
3619     @Override
3620     public ArrayList<ERROR_CODE> getErrorList() {
3621       return errorList;
3622     }
3623 
3624     @Override
3625     public synchronized void print(String message) {
3626       if (!summary) {
3627         System.out.println(message);
3628       }
3629     }
3630 
3631     @Override
3632     public boolean tableHasErrors(TableInfo table) {
3633       return errorTables.contains(table);
3634     }
3635 
3636     @Override
3637     public void resetErrors() {
3638       errorCount = 0;
3639     }
3640 
3641     @Override
3642     public synchronized void detail(String message) {
3643       if (details) {
3644         System.out.println(message);
3645       }
3646       showProgress = 0;
3647     }
3648 
3649     @Override
3650     public synchronized void progress() {
3651       if (showProgress++ == 10) {
3652         if (!summary) {
3653           System.out.print(".");
3654         }
3655         showProgress = 0;
3656       }
3657     }
3658   }
3659 
3660   /**
3661    * Contact a region server and get all information from it
3662    */
3663   static class WorkItemRegion implements Callable<Void> {
3664     private HBaseFsck hbck;
3665     private ServerName rsinfo;
3666     private ErrorReporter errors;
3667     private HConnection connection;
3668 
3669     WorkItemRegion(HBaseFsck hbck, ServerName info,
3670                    ErrorReporter errors, HConnection connection) {
3671       this.hbck = hbck;
3672       this.rsinfo = info;
3673       this.errors = errors;
3674       this.connection = connection;
3675     }
3676 
3677     @Override
3678     public synchronized Void call() throws IOException {
3679       errors.progress();
3680       try {
3681         BlockingInterface server = connection.getAdmin(rsinfo);
3682 
3683         // list all online regions from this region server
3684         List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
3685         regions = filterRegions(regions);
3686 
3687         if (details) {
3688           errors.detail("RegionServer: " + rsinfo.getServerName() +
3689                            " number of regions: " + regions.size());
3690           for (HRegionInfo rinfo: regions) {
3691             errors.detail("  " + rinfo.getRegionNameAsString() +
3692                              " id: " + rinfo.getRegionId() +
3693                              " encoded_name: " + rinfo.getEncodedName() +
3694                              " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
3695                              " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
3696           }
3697         }
3698 
3699         // check to see if the existence of this region matches the region in META
3700         for (HRegionInfo r:regions) {
3701           HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
3702           hbi.addServer(r, rsinfo);
3703         }
3704       } catch (IOException e) {          // unable to connect to the region server.
3705         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
3706           " Unable to fetch region information. " + e);
3707         throw e;
3708       }
3709       return null;
3710     }
3711 
3712     private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
3713       List<HRegionInfo> ret = Lists.newArrayList();
3714       for (HRegionInfo hri : regions) {
3715         if (hri.isMetaTable() || (!hbck.checkMetaOnly
3716             && hbck.isTableIncluded(hri.getTable()))) {
3717           ret.add(hri);
3718         }
3719       }
3720       return ret;
3721     }
3722   }
3723 
3724   /**
3725    * Contact hdfs and get all information about specified table directory into
3726    * regioninfo list.
3727    */
3728   static class WorkItemHdfsDir implements Callable<Void> {
3729     private HBaseFsck hbck;
3730     private FileStatus tableDir;
3731     private ErrorReporter errors;
3732     private FileSystem fs;
3733 
3734     WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
3735                     FileStatus status) {
3736       this.hbck = hbck;
3737       this.fs = fs;
3738       this.tableDir = status;
3739       this.errors = errors;
3740     }
3741 
3742     @Override
3743     public synchronized Void call() throws IOException {
3744       try {
3745         // level 2: <HBASE_DIR>/<table>/*
3746         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
3747         for (FileStatus regionDir : regionDirs) {
3748           String encodedName = regionDir.getPath().getName();
3749           // ignore directories that aren't hexadecimal
3750           if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
3751             continue;
3752           }
3753 
3754           LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
3755           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
3756           HdfsEntry he = new HdfsEntry();
3757           synchronized (hbi) {
3758             if (hbi.getHdfsRegionDir() != null) {
3759               errors.print("Directory " + encodedName + " duplicate??" +
3760                            hbi.getHdfsRegionDir());
3761             }
3762 
3763             he.hdfsRegionDir = regionDir.getPath();
3764             he.hdfsRegionDirModTime = regionDir.getModificationTime();
3765             Path regioninfoFile = new Path(he.hdfsRegionDir, HRegionFileSystem.REGION_INFO_FILE);
3766             he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
3767             // we add to orphan list when we attempt to read .regioninfo
3768 
3769             // Set a flag if this region contains only edits
3770             // This is special case if a region is left after split
3771             he.hdfsOnlyEdits = true;
3772             FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
3773             Path ePath = WALSplitter.getRegionDirRecoveredEditsDir(regionDir.getPath());
3774             for (FileStatus subDir : subDirs) {
3775               String sdName = subDir.getPath().getName();
3776               if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
3777                 he.hdfsOnlyEdits = false;
3778                 break;
3779               }
3780             }
3781             hbi.hdfsEntry = he;
3782           }
3783         }
3784       } catch (IOException e) {
3785         // unable to connect to the region server.
3786         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
3787             + tableDir.getPath().getName()
3788             + " Unable to fetch region information. " + e);
3789         throw e;
3790       }
3791       return null;
3792     }
3793   }
3794 
3795   /**
3796    * Contact hdfs and get all information about specified table directory into
3797    * regioninfo list.
3798    */
3799   static class WorkItemHdfsRegionInfo implements Callable<Void> {
3800     private HbckInfo hbi;
3801     private HBaseFsck hbck;
3802     private ErrorReporter errors;
3803 
3804     WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
3805       this.hbi = hbi;
3806       this.hbck = hbck;
3807       this.errors = errors;
3808     }
3809 
3810     @Override
3811     public synchronized Void call() throws IOException {
3812       // only load entries that haven't been loaded yet.
3813       if (hbi.getHdfsHRI() == null) {
3814         try {
3815           hbck.loadHdfsRegioninfo(hbi);
3816         } catch (IOException ioe) {
3817           String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
3818               + hbi.getTableName() + " in hdfs dir "
3819               + hbi.getHdfsRegionDir()
3820               + "!  It may be an invalid format or version file.  Treating as "
3821               + "an orphaned regiondir.";
3822           errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
3823           try {
3824             hbck.debugLsr(hbi.getHdfsRegionDir());
3825           } catch (IOException ioe2) {
3826             LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
3827             throw ioe2;
3828           }
3829           hbck.orphanHdfsDirs.add(hbi);
3830           throw ioe;
3831         }
3832       }
3833       return null;
3834     }
3835   };
3836 
3837   /**
3838    * Display the full report from fsck. This displays all live and dead region
3839    * servers, and all known regions.
3840    */
3841   public void setDisplayFullReport() {
3842     details = true;
3843   }
3844 
3845   /**
3846    * Set summary mode.
3847    * Print only summary of the tables and status (OK or INCONSISTENT)
3848    */
3849   void setSummary() {
3850     summary = true;
3851   }
3852 
3853   /**
3854    * Set hbase:meta check mode.
3855    * Print only info about hbase:meta table deployment/state
3856    */
3857   void setCheckMetaOnly() {
3858     checkMetaOnly = true;
3859   }
3860 
3861   /**
3862    * Set region boundaries check mode.
3863    */
3864   void setRegionBoundariesCheck() {
3865     checkRegionBoundaries = true;
3866   }
3867 
3868   /**
3869    * Set table locks fix mode.
3870    * Delete table locks held for a long time
3871    */
3872   public void setFixTableLocks(boolean shouldFix) {
3873     fixTableLocks = shouldFix;
3874     fixAny |= shouldFix;
3875   }
3876 
3877   /**
3878    * Check if we should rerun fsck again. This checks if we've tried to
3879    * fix something and we should rerun fsck tool again.
3880    * Display the full report from fsck. This displays all live and dead
3881    * region servers, and all known regions.
3882    */
3883   void setShouldRerun() {
3884     rerun = true;
3885   }
3886 
3887   boolean shouldRerun() {
3888     return rerun;
3889   }
3890 
3891   /**
3892    * Fix inconsistencies found by fsck. This should try to fix errors (if any)
3893    * found by fsck utility.
3894    */
3895   public void setFixAssignments(boolean shouldFix) {
3896     fixAssignments = shouldFix;
3897     fixAny |= shouldFix;
3898   }
3899 
3900   boolean shouldFixAssignments() {
3901     return fixAssignments;
3902   }
3903 
3904   public void setFixMeta(boolean shouldFix) {
3905     fixMeta = shouldFix;
3906     fixAny |= shouldFix;
3907   }
3908 
3909   boolean shouldFixMeta() {
3910     return fixMeta;
3911   }
3912 
3913   public void setFixEmptyMetaCells(boolean shouldFix) {
3914     fixEmptyMetaCells = shouldFix;
3915     fixAny |= shouldFix;
3916   }
3917 
3918   boolean shouldFixEmptyMetaCells() {
3919     return fixEmptyMetaCells;
3920   }
3921 
3922   public void setCheckHdfs(boolean checking) {
3923     checkHdfs = checking;
3924   }
3925 
3926   boolean shouldCheckHdfs() {
3927     return checkHdfs;
3928   }
3929 
3930   public void setFixHdfsHoles(boolean shouldFix) {
3931     fixHdfsHoles = shouldFix;
3932     fixAny |= shouldFix;
3933   }
3934 
3935   boolean shouldFixHdfsHoles() {
3936     return fixHdfsHoles;
3937   }
3938 
3939   public void setFixTableOrphans(boolean shouldFix) {
3940     fixTableOrphans = shouldFix;
3941     fixAny |= shouldFix;
3942   }
3943 
3944   boolean shouldFixTableOrphans() {
3945     return fixTableOrphans;
3946   }
3947 
3948   public void setFixHdfsOverlaps(boolean shouldFix) {
3949     fixHdfsOverlaps = shouldFix;
3950     fixAny |= shouldFix;
3951   }
3952 
3953   boolean shouldFixHdfsOverlaps() {
3954     return fixHdfsOverlaps;
3955   }
3956 
3957   public void setFixHdfsOrphans(boolean shouldFix) {
3958     fixHdfsOrphans = shouldFix;
3959     fixAny |= shouldFix;
3960   }
3961 
3962   boolean shouldFixHdfsOrphans() {
3963     return fixHdfsOrphans;
3964   }
3965 
3966   public void setFixVersionFile(boolean shouldFix) {
3967     fixVersionFile = shouldFix;
3968     fixAny |= shouldFix;
3969   }
3970 
3971   public boolean shouldFixVersionFile() {
3972     return fixVersionFile;
3973   }
3974 
3975   public void setSidelineBigOverlaps(boolean sbo) {
3976     this.sidelineBigOverlaps = sbo;
3977   }
3978 
3979   public boolean shouldSidelineBigOverlaps() {
3980     return sidelineBigOverlaps;
3981   }
3982 
3983   public void setFixSplitParents(boolean shouldFix) {
3984     fixSplitParents = shouldFix;
3985     fixAny |= shouldFix;
3986   }
3987 
3988   boolean shouldFixSplitParents() {
3989     return fixSplitParents;
3990   }
3991 
3992   public void setFixReferenceFiles(boolean shouldFix) {
3993     fixReferenceFiles = shouldFix;
3994     fixAny |= shouldFix;
3995   }
3996 
3997   boolean shouldFixReferenceFiles() {
3998     return fixReferenceFiles;
3999   }
4000 
4001   public boolean shouldIgnorePreCheckPermission() {
4002     return !fixAny || ignorePreCheckPermission;
4003   }
4004 
4005   public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
4006     this.ignorePreCheckPermission = ignorePreCheckPermission;
4007   }
4008 
4009   /**
4010    * @param mm maximum number of regions to merge into a single region.
4011    */
4012   public void setMaxMerge(int mm) {
4013     this.maxMerge = mm;
4014   }
4015 
4016   public int getMaxMerge() {
4017     return maxMerge;
4018   }
4019 
4020   public void setMaxOverlapsToSideline(int mo) {
4021     this.maxOverlapsToSideline = mo;
4022   }
4023 
4024   public int getMaxOverlapsToSideline() {
4025     return maxOverlapsToSideline;
4026   }
4027 
4028   /**
4029    * Only check/fix tables specified by the list,
4030    * Empty list means all tables are included.
4031    */
4032   boolean isTableIncluded(TableName table) {
4033     return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
4034   }
4035 
4036   public void includeTable(TableName table) {
4037     tablesIncluded.add(table);
4038   }
4039 
4040   Set<TableName> getIncludedTables() {
4041     return new HashSet<TableName>(tablesIncluded);
4042   }
4043 
4044   /**
4045    * We are interested in only those tables that have not changed their state in
4046    * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
4047    * @param seconds - the time in seconds
4048    */
4049   public void setTimeLag(long seconds) {
4050     timelag = seconds * 1000; // convert to milliseconds
4051   }
4052 
4053   /**
4054    *
4055    * @param sidelineDir - HDFS path to sideline data
4056    */
4057   public void setSidelineDir(String sidelineDir) {
4058     this.sidelineDir = new Path(sidelineDir);
4059   }
4060 
4061   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
4062     return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
4063   }
4064 
4065   public HFileCorruptionChecker getHFilecorruptionChecker() {
4066     return hfcc;
4067   }
4068 
4069   public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
4070     this.hfcc = hfcc;
4071   }
4072 
4073   public void setRetCode(int code) {
4074     this.retcode = code;
4075   }
4076 
4077   public int getRetCode() {
4078     return retcode;
4079   }
4080 
4081   protected HBaseFsck printUsageAndExit() {
4082     StringWriter sw = new StringWriter(2048);
4083     PrintWriter out = new PrintWriter(sw);
4084     out.println("Usage: fsck [opts] {only tables}");
4085     out.println(" where [opts] are:");
4086     out.println("   -help Display help options (this)");
4087     out.println("   -details Display full report of all regions.");
4088     out.println("   -timelag <timeInSeconds>  Process only regions that " +
4089                        " have not experienced any metadata updates in the last " +
4090                        " <timeInSeconds> seconds.");
4091     out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
4092         " before checking if the fix worked if run with -fix");
4093     out.println("   -summary Print only summary of the tables and status.");
4094     out.println("   -metaonly Only check the state of the hbase:meta table.");
4095     out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
4096     out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
4097 
4098     out.println("");
4099     out.println("  Metadata Repair options: (expert features, use with caution!)");
4100     out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
4101     out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
4102     out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
4103     out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
4104         + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
4105     out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
4106     out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
4107     out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
4108     out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
4109     out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
4110     out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
4111     out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
4112     out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
4113     out.println("   -fixSplitParents  Try to force offline split parents to be online.");
4114     out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
4115     out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
4116     out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
4117         + " (empty REGIONINFO_QUALIFIER rows)");
4118 
4119     out.println("");
4120     out.println("  Datafile Repair options: (expert features, use with caution!)");
4121     out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
4122     out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
4123 
4124     out.println("");
4125     out.println("  Metadata Repair shortcuts");
4126     out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
4127         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles -fixTableLocks");
4128     out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
4129 
4130     out.println("");
4131     out.println("  Table lock options");
4132     out.println("   -fixTableLocks    Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
4133 
4134     out.flush();
4135     errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
4136 
4137     setRetCode(-2);
4138     return this;
4139   }
4140 
4141   /**
4142    * Main program
4143    *
4144    * @param args
4145    * @throws Exception
4146    */
4147   public static void main(String[] args) throws Exception {
4148     // create a fsck object
4149     Configuration conf = HBaseConfiguration.create();
4150     Path hbasedir = FSUtils.getRootDir(conf);
4151     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
4152     FSUtils.setFsDefault(conf, new Path(defaultFs));
4153     int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
4154     System.exit(ret);
4155   }
4156 
4157   /**
4158    * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
4159    */
4160   static class HBaseFsckTool extends Configured implements Tool {
4161     HBaseFsckTool(Configuration conf) { super(conf); }
4162     @Override
4163     public int run(String[] args) throws Exception {
4164       HBaseFsck hbck = new HBaseFsck(getConf());
4165       hbck.exec(hbck.executor, args);
4166       hbck.close();
4167       return hbck.getRetCode();
4168     }
4169   };
4170 
4171 
4172   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
4173     ServiceException, InterruptedException {
4174     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
4175 
4176     boolean checkCorruptHFiles = false;
4177     boolean sidelineCorruptHFiles = false;
4178 
4179     // Process command-line args.
4180     for (int i = 0; i < args.length; i++) {
4181       String cmd = args[i];
4182       if (cmd.equals("-help") || cmd.equals("-h")) {
4183         return printUsageAndExit();
4184       } else if (cmd.equals("-details")) {
4185         setDisplayFullReport();
4186       } else if (cmd.equals("-timelag")) {
4187         if (i == args.length - 1) {
4188           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4189           return printUsageAndExit();
4190         }
4191         try {
4192           long timelag = Long.parseLong(args[i+1]);
4193           setTimeLag(timelag);
4194         } catch (NumberFormatException e) {
4195           errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4196           return printUsageAndExit();
4197         }
4198         i++;
4199       } else if (cmd.equals("-sleepBeforeRerun")) {
4200         if (i == args.length - 1) {
4201           errors.reportError(ERROR_CODE.WRONG_USAGE,
4202             "HBaseFsck: -sleepBeforeRerun needs a value.");
4203           return printUsageAndExit();
4204         }
4205         try {
4206           sleepBeforeRerun = Long.parseLong(args[i+1]);
4207         } catch (NumberFormatException e) {
4208           errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4209           return printUsageAndExit();
4210         }
4211         i++;
4212       } else if (cmd.equals("-sidelineDir")) {
4213         if (i == args.length - 1) {
4214           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4215           return printUsageAndExit();
4216         }
4217         i++;
4218         setSidelineDir(args[i]);
4219       } else if (cmd.equals("-fix")) {
4220         errors.reportError(ERROR_CODE.WRONG_USAGE,
4221           "This option is deprecated, please use  -fixAssignments instead.");
4222         setFixAssignments(true);
4223       } else if (cmd.equals("-fixAssignments")) {
4224         setFixAssignments(true);
4225       } else if (cmd.equals("-fixMeta")) {
4226         setFixMeta(true);
4227       } else if (cmd.equals("-noHdfsChecking")) {
4228         setCheckHdfs(false);
4229       } else if (cmd.equals("-fixHdfsHoles")) {
4230         setFixHdfsHoles(true);
4231       } else if (cmd.equals("-fixHdfsOrphans")) {
4232         setFixHdfsOrphans(true);
4233       } else if (cmd.equals("-fixTableOrphans")) {
4234         setFixTableOrphans(true);
4235       } else if (cmd.equals("-fixHdfsOverlaps")) {
4236         setFixHdfsOverlaps(true);
4237       } else if (cmd.equals("-fixVersionFile")) {
4238         setFixVersionFile(true);
4239       } else if (cmd.equals("-sidelineBigOverlaps")) {
4240         setSidelineBigOverlaps(true);
4241       } else if (cmd.equals("-fixSplitParents")) {
4242         setFixSplitParents(true);
4243       } else if (cmd.equals("-ignorePreCheckPermission")) {
4244         setIgnorePreCheckPermission(true);
4245       } else if (cmd.equals("-checkCorruptHFiles")) {
4246         checkCorruptHFiles = true;
4247       } else if (cmd.equals("-sidelineCorruptHFiles")) {
4248         sidelineCorruptHFiles = true;
4249       } else if (cmd.equals("-fixReferenceFiles")) {
4250         setFixReferenceFiles(true);
4251       } else if (cmd.equals("-fixEmptyMetaCells")) {
4252         setFixEmptyMetaCells(true);
4253       } else if (cmd.equals("-repair")) {
4254         // this attempts to merge overlapping hdfs regions, needs testing
4255         // under load
4256         setFixHdfsHoles(true);
4257         setFixHdfsOrphans(true);
4258         setFixMeta(true);
4259         setFixAssignments(true);
4260         setFixHdfsOverlaps(true);
4261         setFixVersionFile(true);
4262         setSidelineBigOverlaps(true);
4263         setFixSplitParents(false);
4264         setCheckHdfs(true);
4265         setFixReferenceFiles(true);
4266         setFixTableLocks(true);
4267       } else if (cmd.equals("-repairHoles")) {
4268         // this will make all missing hdfs regions available but may lose data
4269         setFixHdfsHoles(true);
4270         setFixHdfsOrphans(false);
4271         setFixMeta(true);
4272         setFixAssignments(true);
4273         setFixHdfsOverlaps(false);
4274         setSidelineBigOverlaps(false);
4275         setFixSplitParents(false);
4276         setCheckHdfs(true);
4277       } else if (cmd.equals("-maxOverlapsToSideline")) {
4278         if (i == args.length - 1) {
4279           errors.reportError(ERROR_CODE.WRONG_USAGE,
4280             "-maxOverlapsToSideline needs a numeric value argument.");
4281           return printUsageAndExit();
4282         }
4283         try {
4284           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4285           setMaxOverlapsToSideline(maxOverlapsToSideline);
4286         } catch (NumberFormatException e) {
4287           errors.reportError(ERROR_CODE.WRONG_USAGE,
4288             "-maxOverlapsToSideline needs a numeric value argument.");
4289           return printUsageAndExit();
4290         }
4291         i++;
4292       } else if (cmd.equals("-maxMerge")) {
4293         if (i == args.length - 1) {
4294           errors.reportError(ERROR_CODE.WRONG_USAGE,
4295             "-maxMerge needs a numeric value argument.");
4296           return printUsageAndExit();
4297         }
4298         try {
4299           int maxMerge = Integer.parseInt(args[i+1]);
4300           setMaxMerge(maxMerge);
4301         } catch (NumberFormatException e) {
4302           errors.reportError(ERROR_CODE.WRONG_USAGE,
4303             "-maxMerge needs a numeric value argument.");
4304           return printUsageAndExit();
4305         }
4306         i++;
4307       } else if (cmd.equals("-summary")) {
4308         setSummary();
4309       } else if (cmd.equals("-metaonly")) {
4310         setCheckMetaOnly();
4311       } else if (cmd.equals("-boundaries")) {
4312         setRegionBoundariesCheck();
4313       } else if (cmd.equals("-fixTableLocks")) {
4314         setFixTableLocks(true);
4315       } else if (cmd.startsWith("-")) {
4316         errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4317         return printUsageAndExit();
4318       } else {
4319         includeTable(TableName.valueOf(cmd));
4320         errors.print("Allow checking/fixes for table: " + cmd);
4321       }
4322     }
4323 
4324     errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
4325 
4326     // pre-check current user has FS write permission or not
4327     try {
4328       preCheckPermission();
4329     } catch (AccessDeniedException ace) {
4330       Runtime.getRuntime().exit(-1);
4331     } catch (IOException ioe) {
4332       Runtime.getRuntime().exit(-1);
4333     }
4334 
4335     // do the real work of hbck
4336     connect();
4337 
4338     try {
4339       // if corrupt file mode is on, first fix them since they may be opened later
4340       if (checkCorruptHFiles || sidelineCorruptHFiles) {
4341         LOG.info("Checking all hfiles for corruption");
4342         HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4343         setHFileCorruptionChecker(hfcc); // so we can get result
4344         Collection<TableName> tables = getIncludedTables();
4345         Collection<Path> tableDirs = new ArrayList<Path>();
4346         Path rootdir = FSUtils.getRootDir(getConf());
4347         if (tables.size() > 0) {
4348           for (TableName t : tables) {
4349             tableDirs.add(FSUtils.getTableDir(rootdir, t));
4350           }
4351         } else {
4352           tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4353         }
4354         hfcc.checkTables(tableDirs);
4355         hfcc.report(errors);
4356       }
4357 
4358       // check and fix table integrity, region consistency.
4359       int code = onlineHbck();
4360       setRetCode(code);
4361       // If we have changed the HBase state it is better to run hbck again
4362       // to see if we haven't broken something else in the process.
4363       // We run it only once more because otherwise we can easily fall into
4364       // an infinite loop.
4365       if (shouldRerun()) {
4366         try {
4367           LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4368           Thread.sleep(sleepBeforeRerun);
4369         } catch (InterruptedException ie) {
4370           LOG.warn("Interrupted while sleeping");
4371           return this;
4372         }
4373         // Just report
4374         setFixAssignments(false);
4375         setFixMeta(false);
4376         setFixHdfsHoles(false);
4377         setFixHdfsOverlaps(false);
4378         setFixVersionFile(false);
4379         setFixTableOrphans(false);
4380         errors.resetErrors();
4381         code = onlineHbck();
4382         setRetCode(code);
4383       }
4384     } finally {
4385       IOUtils.cleanup(null, this);
4386     }
4387     return this;
4388   }
4389 
4390   /**
4391    * ls -r for debugging purposes
4392    */
4393   void debugLsr(Path p) throws IOException {
4394     debugLsr(getConf(), p, errors);
4395   }
4396 
4397   /**
4398    * ls -r for debugging purposes
4399    */
4400   public static void debugLsr(Configuration conf,
4401       Path p) throws IOException {
4402     debugLsr(conf, p, new PrintingErrorReporter());
4403   }
4404 
4405   /**
4406    * ls -r for debugging purposes
4407    */
4408   public static void debugLsr(Configuration conf,
4409       Path p, ErrorReporter errors) throws IOException {
4410     if (!LOG.isDebugEnabled() || p == null) {
4411       return;
4412     }
4413     FileSystem fs = p.getFileSystem(conf);
4414 
4415     if (!fs.exists(p)) {
4416       // nothing
4417       return;
4418     }
4419     errors.print(p.toString());
4420 
4421     if (fs.isFile(p)) {
4422       return;
4423     }
4424 
4425     if (fs.getFileStatus(p).isDirectory()) {
4426       FileStatus[] fss= fs.listStatus(p);
4427       for (FileStatus status : fss) {
4428         debugLsr(conf, status.getPath(), errors);
4429       }
4430     }
4431   }
4432 }