View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.FileNotFoundException;
21  import java.io.IOException;
22  import java.io.PrintWriter;
23  import java.io.StringWriter;
24  import java.net.InetAddress;
25  import java.net.URI;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.Collection;
29  import java.util.Collections;
30  import java.util.Comparator;
31  import java.util.HashMap;
32  import java.util.HashSet;
33  import java.util.Iterator;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.Map.Entry;
37  import java.util.Set;
38  import java.util.SortedMap;
39  import java.util.SortedSet;
40  import java.util.TreeMap;
41  import java.util.TreeSet;
42  import java.util.concurrent.Callable;
43  import java.util.concurrent.ConcurrentSkipListMap;
44  import java.util.concurrent.ExecutionException;
45  import java.util.concurrent.ExecutorService;
46  import java.util.concurrent.Future;
47  import java.util.concurrent.ScheduledThreadPoolExecutor;
48  import java.util.concurrent.atomic.AtomicBoolean;
49  import java.util.concurrent.atomic.AtomicInteger;
50  
51  
52  import org.apache.commons.logging.Log;
53  import org.apache.commons.logging.LogFactory;
54  import org.apache.hadoop.hbase.classification.InterfaceAudience;
55  import org.apache.hadoop.hbase.classification.InterfaceStability;
56  import org.apache.hadoop.conf.Configuration;
57  import org.apache.hadoop.conf.Configured;
58  import org.apache.hadoop.fs.FSDataOutputStream;
59  import org.apache.hadoop.fs.FileStatus;
60  import org.apache.hadoop.fs.FileSystem;
61  import org.apache.hadoop.fs.Path;
62  import org.apache.hadoop.fs.permission.FsAction;
63  import org.apache.hadoop.fs.permission.FsPermission;
64  import org.apache.hadoop.hbase.Abortable;
65  import org.apache.hadoop.hbase.Cell;
66  import org.apache.hadoop.hbase.ClusterStatus;
67  import org.apache.hadoop.hbase.HBaseConfiguration;
68  import org.apache.hadoop.hbase.HColumnDescriptor;
69  import org.apache.hadoop.hbase.HConstants;
70  import org.apache.hadoop.hbase.HRegionInfo;
71  import org.apache.hadoop.hbase.HRegionLocation;
72  import org.apache.hadoop.hbase.HTableDescriptor;
73  import org.apache.hadoop.hbase.KeyValue;
74  import org.apache.hadoop.hbase.MasterNotRunningException;
75  import org.apache.hadoop.hbase.MetaTableAccessor;
76  import org.apache.hadoop.hbase.RegionLocations;
77  import org.apache.hadoop.hbase.ServerName;
78  import org.apache.hadoop.hbase.TableDescriptor;
79  import org.apache.hadoop.hbase.TableName;
80  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
81  import org.apache.hadoop.hbase.client.Admin;
82  import org.apache.hadoop.hbase.client.Delete;
83  import org.apache.hadoop.hbase.client.Get;
84  import org.apache.hadoop.hbase.client.HBaseAdmin;
85  import org.apache.hadoop.hbase.client.HConnectable;
86  import org.apache.hadoop.hbase.client.HConnection;
87  import org.apache.hadoop.hbase.client.HConnectionManager;
88  import org.apache.hadoop.hbase.client.HTable;
89  import org.apache.hadoop.hbase.client.MetaScanner;
90  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
91  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
92  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
93  import org.apache.hadoop.hbase.client.Put;
94  import org.apache.hadoop.hbase.client.Result;
95  import org.apache.hadoop.hbase.client.RowMutations;
96  import org.apache.hadoop.hbase.client.Table;
97  import org.apache.hadoop.hbase.client.TableState;
98  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
99  import org.apache.hadoop.hbase.io.hfile.HFile;
100 import org.apache.hadoop.hbase.master.MasterFileSystem;
101 import org.apache.hadoop.hbase.master.RegionState;
102 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
103 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
104 import org.apache.hadoop.hbase.regionserver.HRegion;
105 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
106 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
107 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
108 import org.apache.hadoop.hbase.security.UserProvider;
109 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
110 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
111 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
112 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
113 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
114 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
115 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
116 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
117 import org.apache.hadoop.hbase.security.AccessDeniedException;
118 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
119 import org.apache.hadoop.io.IOUtils;
120 import org.apache.hadoop.ipc.RemoteException;
121 import org.apache.hadoop.security.UserGroupInformation;
122 import org.apache.hadoop.util.ReflectionUtils;
123 import org.apache.hadoop.util.Tool;
124 import org.apache.hadoop.util.ToolRunner;
125 import org.apache.zookeeper.KeeperException;
126 
127 import com.google.common.base.Joiner;
128 import com.google.common.base.Preconditions;
129 import com.google.common.collect.Lists;
130 import com.google.common.collect.Multimap;
131 import com.google.common.collect.TreeMultimap;
132 import com.google.protobuf.ServiceException;
133 
134 /**
135  * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
136  * table integrity problems in a corrupted HBase.
137  * <p>
138  * Region consistency checks verify that hbase:meta, region deployment on region
139  * servers and the state of data in HDFS (.regioninfo files) all are in
140  * accordance.
141  * <p>
142  * Table integrity checks verify that all possible row keys resolve to exactly
143  * one region of a table.  This means there are no individual degenerate
144  * or backwards regions; no holes between regions; and that there are no
145  * overlapping regions.
146  * <p>
147  * The general repair strategy works in two phases:
148  * <ol>
149  * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
150  * <li> Repair Region Consistency with hbase:meta and assignments
151  * </ol>
152  * <p>
153  * For table integrity repairs, the tables' region directories are scanned
154  * for .regioninfo files.  Each table's integrity is then verified.  If there
155  * are any orphan regions (regions with no .regioninfo files) or holes, new
156  * regions are fabricated.  Backwards regions are sidelined as well as empty
157  * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
158  * a new region is created and all data is merged into the new region.
159  * <p>
160  * Table integrity repairs deal solely with HDFS and could potentially be done
161  * offline -- the hbase region servers or master do not need to be running.
162  * This phase can eventually be used to completely reconstruct the hbase:meta table in
163  * an offline fashion.
164  * <p>
165  * Region consistency requires three conditions -- 1) valid .regioninfo file
166  * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
167  * and 3) a region is deployed only at the regionserver that was assigned to
168  * with proper state in the master.
169  * <p>
170  * Region consistency repairs require hbase to be online so that hbck can
171  * contact the HBase master and region servers.  The hbck#connect() method must
172  * first be called successfully.  Much of the region consistency information
173  * is transient and less risky to repair.
174  * <p>
175  * If hbck is run from the command line, there are a handful of arguments that
176  * can be used to limit the kinds of repairs hbck will do.  See the code in
177  * {@link #printUsageAndExit()} for more details.
178  */
179 @InterfaceAudience.Public
180 @InterfaceStability.Evolving
181 public class HBaseFsck extends Configured {
182   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
183   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
184   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
185   private static boolean rsSupportsOffline = true;
186   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
187   private static final int DEFAULT_MAX_MERGE = 5;
188   private static final String TO_BE_LOADED = "to_be_loaded";
189   private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
190 
191 
192   /**********************
193    * Internal resources
194    **********************/
195   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
196   private ClusterStatus status;
197   private HConnection connection;
198   private HBaseAdmin admin;
199   private Table meta;
200   // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
201   protected ExecutorService executor;
202   private long startMillis = System.currentTimeMillis();
203   private HFileCorruptionChecker hfcc;
204   private int retcode = 0;
205   private Path HBCK_LOCK_PATH;
206   private FSDataOutputStream hbckOutFd;
207   // This lock is to prevent cleanup of balancer resources twice between
208   // ShutdownHook and the main code. We cleanup only if the connect() is
209   // successful
210   private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
211 
212   /***********
213    * Options
214    ***********/
215   private static boolean details = false; // do we display the full report
216   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
217   private boolean fixAssignments = false; // fix assignment errors?
218   private boolean fixMeta = false; // fix meta errors?
219   private boolean checkHdfs = true; // load and check fs consistency?
220   private boolean fixHdfsHoles = false; // fix fs holes?
221   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
222   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
223   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
224   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
225   private boolean fixSplitParents = false; // fix lingering split parents
226   private boolean fixReferenceFiles = false; // fix lingering reference store file
227   private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
228   private boolean fixTableLocks = false; // fix table locks which are expired
229   private boolean fixAny = false; // Set to true if any of the fix is required.
230 
231   // limit checking/fixes to listed tables, if empty attempt to check/fix all
232   // hbase:meta are always checked
233   private Set<TableName> tablesIncluded = new HashSet<TableName>();
234   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
235   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
236   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
237   private Path sidelineDir = null;
238 
239   private boolean rerun = false; // if we tried to fix something, rerun hbck
240   private static boolean summary = false; // if we want to print less output
241   private boolean checkMetaOnly = false;
242   private boolean checkRegionBoundaries = false;
243   private boolean ignorePreCheckPermission = false; // if pre-check permission
244 
245   /*********
246    * State
247    *********/
248   final private ErrorReporter errors;
249   int fixes = 0;
250 
251   /**
252    * This map contains the state of all hbck items.  It maps from encoded region
253    * name to HbckInfo structure.  The information contained in HbckInfo is used
254    * to detect and correct consistency (hdfs/meta/deployment) problems.
255    */
256   private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
257   private TreeSet<TableName> disabledTables =
258     new TreeSet<TableName>();
259   // Empty regioninfo qualifiers in hbase:meta
260   private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
261 
262   /**
263    * This map from Tablename -> TableInfo contains the structures necessary to
264    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
265    * to prevent dupes.
266    *
267    * If tablesIncluded is empty, this map contains all tables.
268    * Otherwise, it contains only meta tables and tables in tablesIncluded,
269    * unless checkMetaOnly is specified, in which case, it contains only
270    * the meta table
271    */
272   private SortedMap<TableName, TableInfo> tablesInfo =
273       new ConcurrentSkipListMap<TableName, TableInfo>();
274 
275   /**
276    * When initially looking at HDFS, we attempt to find any orphaned data.
277    */
278   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
279 
280   private Map<TableName, Set<String>> orphanTableDirs =
281       new HashMap<TableName, Set<String>>();
282 
283   /**
284    * Constructor
285    *
286    * @param conf Configuration object
287    * @throws MasterNotRunningException if the master is not running
288    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
289    */
290   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
291       ZooKeeperConnectionException, IOException, ClassNotFoundException {
292     super(conf);
293     // make a copy, just to be sure we're not overriding someone else's config
294     setConf(HBaseConfiguration.create(getConf()));
295     // disable blockcache for tool invocation, see HBASE-10500
296     getConf().setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0);
297     errors = getErrorReporter(conf);
298 
299     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
300     executor = new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
301   }
302 
303   /**
304    * Constructor
305    *
306    * @param conf
307    *          Configuration object
308    * @throws MasterNotRunningException
309    *           if the master is not running
310    * @throws ZooKeeperConnectionException
311    *           if unable to connect to ZooKeeper
312    */
313   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
314       ZooKeeperConnectionException, IOException, ClassNotFoundException {
315     super(conf);
316     errors = getErrorReporter(getConf());
317     this.executor = exec;
318   }
319 
320   /**
321    * This method maintains a lock using a file. If the creation fails we return null
322    *
323    * @return FSDataOutputStream object corresponding to the newly opened lock file
324    * @throws IOException
325    */
326   private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
327     try {
328       FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
329       FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
330           HConstants.DATA_FILE_UMASK_KEY);
331       Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
332       fs.mkdirs(tmpDir);
333       HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
334       final FSDataOutputStream out = FSUtils.create(fs, HBCK_LOCK_PATH, defaultPerms, false);
335       out.writeBytes(InetAddress.getLocalHost().toString());
336       out.flush();
337       return out;
338     } catch(RemoteException e) {
339       if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
340         return null;
341       } else {
342         throw e;
343       }
344     }
345   }
346 
347   private void unlockHbck() {
348     if(hbckLockCleanup.compareAndSet(true, false)){
349       IOUtils.closeStream(hbckOutFd);
350       try{
351         FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()), HBCK_LOCK_PATH, true);
352       } catch(IOException ioe) {
353         LOG.warn("Failed to delete " + HBCK_LOCK_PATH);
354         LOG.debug(ioe);
355       }
356     }
357   }
358 
359   /**
360    * To repair region consistency, one must call connect() in order to repair
361    * online state.
362    */
363   public void connect() throws IOException {
364 
365     // Check if another instance of balancer is running
366     hbckOutFd = checkAndMarkRunningHbck();
367     if (hbckOutFd == null) {
368       setRetCode(-1);
369       LOG.error("Another instance of hbck is running, exiting this instance.[If you are sure" +
370 		      " no other instance is running, delete the lock file " +
371 		      HBCK_LOCK_PATH + " and rerun the tool]");
372       throw new IOException("Duplicate hbck - Abort");
373     }
374 
375     // Make sure to cleanup the lock
376     hbckLockCleanup.set(true);
377 
378     // Add a shutdown hook to this thread, incase user tries to
379     // kill the hbck with a ctrl-c, we want to cleanup the lock so that
380     // it is available for further calls
381     Runtime.getRuntime().addShutdownHook(new Thread() {
382       @Override
383       public void run() {
384           unlockHbck();
385       }
386     });
387     LOG.debug("Launching hbck");
388 
389     connection = HConnectionManager.createConnection(getConf());
390     admin = new HBaseAdmin(connection);
391     meta = new HTable(TableName.META_TABLE_NAME, connection);
392     status = admin.getClusterStatus();
393   }
394 
395   /**
396    * Get deployed regions according to the region servers.
397    */
398   private void loadDeployedRegions() throws IOException, InterruptedException {
399     // From the master, get a list of all known live region servers
400     Collection<ServerName> regionServers = status.getServers();
401     errors.print("Number of live region servers: " + regionServers.size());
402     if (details) {
403       for (ServerName rsinfo: regionServers) {
404         errors.print("  " + rsinfo.getServerName());
405       }
406     }
407 
408     // From the master, get a list of all dead region servers
409     Collection<ServerName> deadRegionServers = status.getDeadServerNames();
410     errors.print("Number of dead region servers: " + deadRegionServers.size());
411     if (details) {
412       for (ServerName name: deadRegionServers) {
413         errors.print("  " + name);
414       }
415     }
416 
417     // Print the current master name and state
418     errors.print("Master: " + status.getMaster());
419 
420     // Print the list of all backup masters
421     Collection<ServerName> backupMasters = status.getBackupMasters();
422     errors.print("Number of backup masters: " + backupMasters.size());
423     if (details) {
424       for (ServerName name: backupMasters) {
425         errors.print("  " + name);
426       }
427     }
428 
429     errors.print("Average load: " + status.getAverageLoad());
430     errors.print("Number of requests: " + status.getRequestsCount());
431     errors.print("Number of regions: " + status.getRegionsCount());
432 
433     Map<String, RegionState> rits = status.getRegionsInTransition();
434     errors.print("Number of regions in transition: " + rits.size());
435     if (details) {
436       for (RegionState state: rits.values()) {
437         errors.print("  " + state.toDescriptiveString());
438       }
439     }
440 
441     // Determine what's deployed
442     processRegionServers(regionServers);
443   }
444 
445   /**
446    * Clear the current state of hbck.
447    */
448   private void clearState() {
449     // Make sure regionInfo is empty before starting
450     fixes = 0;
451     regionInfoMap.clear();
452     emptyRegionInfoQualifiers.clear();
453     disabledTables.clear();
454     errors.clear();
455     tablesInfo.clear();
456     orphanHdfsDirs.clear();
457   }
458 
459   /**
460    * This repair method analyzes hbase data in hdfs and repairs it to satisfy
461    * the table integrity rules.  HBase doesn't need to be online for this
462    * operation to work.
463    */
464   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
465     // Initial pass to fix orphans.
466     if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
467         || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
468       LOG.info("Loading regioninfos HDFS");
469       // if nothing is happening this should always complete in two iterations.
470       int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
471       int curIter = 0;
472       do {
473         clearState(); // clears hbck state and reset fixes to 0 and.
474         // repair what's on HDFS
475         restoreHdfsIntegrity();
476         curIter++;// limit the number of iterations.
477       } while (fixes > 0 && curIter <= maxIterations);
478 
479       // Repairs should be done in the first iteration and verification in the second.
480       // If there are more than 2 passes, something funny has happened.
481       if (curIter > 2) {
482         if (curIter == maxIterations) {
483           LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
484               + "Tables integrity may not be fully repaired!");
485         } else {
486           LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
487         }
488       }
489     }
490   }
491 
492   /**
493    * This repair method requires the cluster to be online since it contacts
494    * region servers and the masters.  It makes each region's state in HDFS, in
495    * hbase:meta, and deployments consistent.
496    *
497    * @return If > 0 , number of errors detected, if < 0 there was an unrecoverable
498    * error.  If 0, we have a clean hbase.
499    */
500   public int onlineConsistencyRepair() throws IOException, KeeperException,
501     InterruptedException {
502     clearState();
503 
504     // get regions according to what is online on each RegionServer
505     loadDeployedRegions();
506     // check whether hbase:meta is deployed and online
507     recordMetaRegion();
508     // Check if hbase:meta is found only once and in the right place
509     if (!checkMetaRegion()) {
510       String errorMsg = "hbase:meta table is not consistent. ";
511       if (shouldFixAssignments()) {
512         errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
513       } else {
514         errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
515       }
516       errors.reportError(errorMsg + " Exiting...");
517       return -2;
518     }
519     // Not going with further consistency check for tables when hbase:meta itself is not consistent.
520     LOG.info("Loading regionsinfo from the hbase:meta table");
521     boolean success = loadMetaEntries();
522     if (!success) return -1;
523 
524     // Empty cells in hbase:meta?
525     reportEmptyMetaCells();
526 
527     // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
528     if (shouldFixEmptyMetaCells()) {
529       fixEmptyMetaCells();
530     }
531 
532     // get a list of all tables that have not changed recently.
533     if (!checkMetaOnly) {
534       reportTablesInFlux();
535     }
536 
537     // load regiondirs and regioninfos from HDFS
538     if (shouldCheckHdfs()) {
539       loadHdfsRegionDirs();
540       loadHdfsRegionInfos();
541     }
542 
543     // Get disabled tables from ZooKeeper
544     loadDisabledTables();
545 
546     // fix the orphan tables
547     fixOrphanTables();
548 
549     // Check and fix consistency
550     checkAndFixConsistency();
551 
552     // Check integrity (does not fix)
553     checkIntegrity();
554     return errors.getErrorList().size();
555   }
556 
557   /**
558    * Contacts the master and prints out cluster-wide information
559    * @return 0 on success, non-zero on failure
560    */
561   public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
562     // print hbase server version
563     errors.print("Version: " + status.getHBaseVersion());
564     offlineHdfsIntegrityRepair();
565 
566     // turn the balancer off
567     boolean oldBalancer = admin.setBalancerRunning(false, true);
568     try {
569       onlineConsistencyRepair();
570     }
571     finally {
572       admin.setBalancerRunning(oldBalancer, false);
573     }
574 
575     if (checkRegionBoundaries) {
576       checkRegionBoundaries();
577     }
578 
579     offlineReferenceFileRepair();
580 
581     checkAndFixTableLocks();
582 
583     // Remove the hbck lock
584     unlockHbck();
585 
586     // Print table summary
587     printTableSummary(tablesInfo);
588     return errors.summarize();
589   }
590 
591   public static byte[] keyOnly (byte[] b) {
592     if (b == null)
593       return b;
594     int rowlength = Bytes.toShort(b, 0);
595     byte[] result = new byte[rowlength];
596     System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
597     return result;
598   }
599 
600   private static class RegionBoundariesInformation {
601     public byte [] regionName;
602     public byte [] metaFirstKey;
603     public byte [] metaLastKey;
604     public byte [] storesFirstKey;
605     public byte [] storesLastKey;
606     @Override
607     public String toString () {
608       return "regionName=" + Bytes.toStringBinary(regionName) +
609              "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
610              "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
611              "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
612              "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
613     }
614   }
615 
616   public void checkRegionBoundaries() {
617     try {
618       ByteArrayComparator comparator = new ByteArrayComparator();
619       List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), false);
620       final RegionBoundariesInformation currentRegionBoundariesInformation =
621           new RegionBoundariesInformation();
622       Path hbaseRoot = FSUtils.getRootDir(getConf());
623       for (HRegionInfo regionInfo : regions) {
624         Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
625         currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
626         // For each region, get the start and stop key from the META and compare them to the
627         // same information from the Stores.
628         Path path = new Path(tableDir, regionInfo.getEncodedName());
629         FileSystem fs = path.getFileSystem(getConf());
630         FileStatus[] files = fs.listStatus(path);
631         // For all the column families in this region...
632         byte[] storeFirstKey = null;
633         byte[] storeLastKey = null;
634         for (FileStatus file : files) {
635           String fileName = file.getPath().toString();
636           fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
637           if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
638             FileStatus[] storeFiles = fs.listStatus(file.getPath());
639             // For all the stores in this column family.
640             for (FileStatus storeFile : storeFiles) {
641               HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
642                   getConf()), getConf());
643               if ((reader.getFirstKey() != null)
644                   && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
645                       reader.getFirstKey()) > 0))) {
646                 storeFirstKey = reader.getFirstKey();
647               }
648               if ((reader.getLastKey() != null)
649                   && ((storeLastKey == null) || (comparator.compare(storeLastKey,
650                       reader.getLastKey())) < 0)) {
651                 storeLastKey = reader.getLastKey();
652               }
653               reader.close();
654             }
655           }
656         }
657         currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
658         currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
659         currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
660         currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
661         if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
662           currentRegionBoundariesInformation.metaFirstKey = null;
663         if (currentRegionBoundariesInformation.metaLastKey.length == 0)
664           currentRegionBoundariesInformation.metaLastKey = null;
665 
666         // For a region to be correct, we need the META start key to be smaller or equal to the
667         // smallest start key from all the stores, and the start key from the next META entry to
668         // be bigger than the last key from all the current stores. First region start key is null;
669         // Last region end key is null; some regions can be empty and not have any store.
670 
671         boolean valid = true;
672         // Checking start key.
673         if ((currentRegionBoundariesInformation.storesFirstKey != null)
674             && (currentRegionBoundariesInformation.metaFirstKey != null)) {
675           valid = valid
676               && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
677                 currentRegionBoundariesInformation.metaFirstKey) >= 0;
678         }
679         // Checking stop key.
680         if ((currentRegionBoundariesInformation.storesLastKey != null)
681             && (currentRegionBoundariesInformation.metaLastKey != null)) {
682           valid = valid
683               && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
684                 currentRegionBoundariesInformation.metaLastKey) < 0;
685         }
686         if (!valid) {
687           errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
688             tablesInfo.get(regionInfo.getTable()));
689           LOG.warn("Region's boundaries not alligned between stores and META for:");
690           LOG.warn(currentRegionBoundariesInformation);
691         }
692       }
693     } catch (IOException e) {
694       LOG.error(e);
695     }
696   }
697 
698   /**
699    * Iterates through the list of all orphan/invalid regiondirs.
700    */
701   private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
702     for (HbckInfo hi : orphanHdfsDirs) {
703       LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
704       adoptHdfsOrphan(hi);
705     }
706   }
707 
708   /**
709    * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
710    * these orphans by creating a new region, and moving the column families,
711    * recovered edits, HLogs, into the new region dir.  We determine the region
712    * startkey and endkeys by looking at all of the hfiles inside the column
713    * families to identify the min and max keys. The resulting region will
714    * likely violate table integrity but will be dealt with by merging
715    * overlapping regions.
716    */
717   @SuppressWarnings("deprecation")
718   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
719     Path p = hi.getHdfsRegionDir();
720     FileSystem fs = p.getFileSystem(getConf());
721     FileStatus[] dirs = fs.listStatus(p);
722     if (dirs == null) {
723       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
724           p + ". This dir could probably be deleted.");
725       return ;
726     }
727 
728     TableName tableName = hi.getTableName();
729     TableInfo tableInfo = tablesInfo.get(tableName);
730     Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
731     HTableDescriptor template = tableInfo.getHTD();
732 
733     // find min and max key values
734     Pair<byte[],byte[]> orphanRegionRange = null;
735     for (FileStatus cf : dirs) {
736       String cfName= cf.getPath().getName();
737       // TODO Figure out what the special dirs are
738       if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
739 
740       FileStatus[] hfiles = fs.listStatus(cf.getPath());
741       for (FileStatus hfile : hfiles) {
742         byte[] start, end;
743         HFile.Reader hf = null;
744         try {
745           CacheConfig cacheConf = new CacheConfig(getConf());
746           hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
747           hf.loadFileInfo();
748           KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
749           start = startKv.getRow();
750           KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
751           end = endKv.getRow();
752         } catch (IOException ioe) {
753           LOG.warn("Problem reading orphan file " + hfile + ", skipping");
754           continue;
755         } catch (NullPointerException ioe) {
756           LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
757           continue;
758         } finally {
759           if (hf != null) {
760             hf.close();
761           }
762         }
763 
764         // expand the range to include the range of all hfiles
765         if (orphanRegionRange == null) {
766           // first range
767           orphanRegionRange = new Pair<byte[], byte[]>(start, end);
768         } else {
769           // TODO add test
770 
771           // expand range only if the hfile is wider.
772           if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
773             orphanRegionRange.setFirst(start);
774           }
775           if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
776             orphanRegionRange.setSecond(end);
777           }
778         }
779       }
780     }
781     if (orphanRegionRange == null) {
782       LOG.warn("No data in dir " + p + ", sidelining data");
783       fixes++;
784       sidelineRegionDir(fs, hi);
785       return;
786     }
787     LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
788         Bytes.toString(orphanRegionRange.getSecond()) + ")");
789 
790     // create new region on hdfs.  move data into place.
791     HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond());
792     LOG.info("Creating new region : " + hri);
793     HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
794     Path target = region.getRegionFileSystem().getRegionDir();
795 
796     // rename all the data to new region
797     mergeRegionDirs(target, hi);
798     fixes++;
799   }
800 
801   /**
802    * This method determines if there are table integrity errors in HDFS.  If
803    * there are errors and the appropriate "fix" options are enabled, the method
804    * will first correct orphan regions making them into legit regiondirs, and
805    * then reload to merge potentially overlapping regions.
806    *
807    * @return number of table integrity errors found
808    */
809   private int restoreHdfsIntegrity() throws IOException, InterruptedException {
810     // Determine what's on HDFS
811     LOG.info("Loading HBase regioninfo from HDFS...");
812     loadHdfsRegionDirs(); // populating regioninfo table.
813 
814     int errs = errors.getErrorList().size();
815     // First time just get suggestions.
816     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
817     checkHdfsIntegrity(false, false);
818 
819     if (errors.getErrorList().size() == errs) {
820       LOG.info("No integrity errors.  We are done with this phase. Glorious.");
821       return 0;
822     }
823 
824     if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
825       adoptHdfsOrphans(orphanHdfsDirs);
826       // TODO optimize by incrementally adding instead of reloading.
827     }
828 
829     // Make sure there are no holes now.
830     if (shouldFixHdfsHoles()) {
831       clearState(); // this also resets # fixes.
832       loadHdfsRegionDirs();
833       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
834       tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
835     }
836 
837     // Now we fix overlaps
838     if (shouldFixHdfsOverlaps()) {
839       // second pass we fix overlaps.
840       clearState(); // this also resets # fixes.
841       loadHdfsRegionDirs();
842       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
843       tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
844     }
845 
846     return errors.getErrorList().size();
847   }
848 
849   /**
850    * Scan all the store file names to find any lingering reference files,
851    * which refer to some none-exiting files. If "fix" option is enabled,
852    * any lingering reference file will be sidelined if found.
853    * <p>
854    * Lingering reference file prevents a region from opening. It has to
855    * be fixed before a cluster can start properly.
856    */
857   private void offlineReferenceFileRepair() throws IOException {
858     Configuration conf = getConf();
859     Path hbaseRoot = FSUtils.getRootDir(conf);
860     FileSystem fs = hbaseRoot.getFileSystem(conf);
861     Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot);
862     for (Path path: allFiles.values()) {
863       boolean isReference = false;
864       try {
865         isReference = StoreFileInfo.isReference(path);
866       } catch (Throwable t) {
867         // Ignore. Some files may not be store files at all.
868         // For example, files under .oldlogs folder in hbase:meta
869         // Warning message is already logged by
870         // StoreFile#isReference.
871       }
872       if (!isReference) continue;
873 
874       Path referredToFile = StoreFileInfo.getReferredToFile(path);
875       if (fs.exists(referredToFile)) continue;  // good, expected
876 
877       // Found a lingering reference file
878       errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
879         "Found lingering reference file " + path);
880       if (!shouldFixReferenceFiles()) continue;
881 
882       // Now, trying to fix it since requested
883       boolean success = false;
884       String pathStr = path.toString();
885 
886       // A reference file path should be like
887       // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
888       // Up 5 directories to get the root folder.
889       // So the file will be sidelined to a similar folder structure.
890       int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
891       for (int i = 0; index > 0 && i < 5; i++) {
892         index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
893       }
894       if (index > 0) {
895         Path rootDir = getSidelineDir();
896         Path dst = new Path(rootDir, pathStr.substring(index + 1));
897         fs.mkdirs(dst.getParent());
898         LOG.info("Trying to sildeline reference file "
899           + path + " to " + dst);
900         setShouldRerun();
901 
902         success = fs.rename(path, dst);
903       }
904       if (!success) {
905         LOG.error("Failed to sideline reference file " + path);
906       }
907     }
908   }
909 
910   /**
911    * TODO -- need to add tests for this.
912    */
913   private void reportEmptyMetaCells() {
914     errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
915       emptyRegionInfoQualifiers.size());
916     if (details) {
917       for (Result r: emptyRegionInfoQualifiers) {
918         errors.print("  " + r);
919       }
920     }
921   }
922 
923   /**
924    * TODO -- need to add tests for this.
925    */
926   private void reportTablesInFlux() {
927     AtomicInteger numSkipped = new AtomicInteger(0);
928     HTableDescriptor[] allTables = getTables(numSkipped);
929     errors.print("Number of Tables: " + allTables.length);
930     if (details) {
931       if (numSkipped.get() > 0) {
932         errors.detail("Number of Tables in flux: " + numSkipped.get());
933       }
934       for (HTableDescriptor td : allTables) {
935         errors.detail("  Table: " + td.getTableName() + "\t" +
936                            (td.isReadOnly() ? "ro" : "rw") + "\t" +
937                             (td.isMetaRegion() ? "META" : "    ") + "\t" +
938                            " families: " + td.getFamilies().size());
939       }
940     }
941   }
942 
943   public ErrorReporter getErrors() {
944     return errors;
945   }
946 
947   /**
948    * Read the .regioninfo file from the file system.  If there is no
949    * .regioninfo, add it to the orphan hdfs region list.
950    */
951   private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
952     Path regionDir = hbi.getHdfsRegionDir();
953     if (regionDir == null) {
954       LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
955       return;
956     }
957 
958     if (hbi.hdfsEntry.hri != null) {
959       // already loaded data
960       return;
961     }
962 
963     FileSystem fs = FileSystem.get(getConf());
964     HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
965     LOG.debug("HRegionInfo read: " + hri.toString());
966     hbi.hdfsEntry.hri = hri;
967   }
968 
969   /**
970    * Exception thrown when a integrity repair operation fails in an
971    * unresolvable way.
972    */
973   public static class RegionRepairException extends IOException {
974     private static final long serialVersionUID = 1L;
975     final IOException ioe;
976     public RegionRepairException(String s, IOException ioe) {
977       super(s);
978       this.ioe = ioe;
979     }
980   }
981 
982   /**
983    * Populate hbi's from regionInfos loaded from file system.
984    */
985   private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
986       throws IOException, InterruptedException {
987     tablesInfo.clear(); // regenerating the data
988     // generate region split structure
989     Collection<HbckInfo> hbckInfos = regionInfoMap.values();
990 
991     // Parallelized read of .regioninfo files.
992     List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
993     List<Future<Void>> hbiFutures;
994 
995     for (HbckInfo hbi : hbckInfos) {
996       WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
997       hbis.add(work);
998     }
999 
1000     // Submit and wait for completion
1001     hbiFutures = executor.invokeAll(hbis);
1002 
1003     for(int i=0; i<hbiFutures.size(); i++) {
1004       WorkItemHdfsRegionInfo work = hbis.get(i);
1005       Future<Void> f = hbiFutures.get(i);
1006       try {
1007         f.get();
1008       } catch(ExecutionException e) {
1009         LOG.warn("Failed to read .regioninfo file for region " +
1010               work.hbi.getRegionNameAsString(), e.getCause());
1011       }
1012     }
1013 
1014     Path hbaseRoot = FSUtils.getRootDir(getConf());
1015     FileSystem fs = hbaseRoot.getFileSystem(getConf());
1016     // serialized table info gathering.
1017     for (HbckInfo hbi: hbckInfos) {
1018 
1019       if (hbi.getHdfsHRI() == null) {
1020         // was an orphan
1021         continue;
1022       }
1023 
1024 
1025       // get table name from hdfs, populate various HBaseFsck tables.
1026       TableName tableName = hbi.getTableName();
1027       if (tableName == null) {
1028         // There was an entry in hbase:meta not in the HDFS?
1029         LOG.warn("tableName was null for: " + hbi);
1030         continue;
1031       }
1032 
1033       TableInfo modTInfo = tablesInfo.get(tableName);
1034       if (modTInfo == null) {
1035         // only executed once per table.
1036         modTInfo = new TableInfo(tableName);
1037         tablesInfo.put(tableName, modTInfo);
1038         try {
1039           TableDescriptor htd =
1040               FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1041           modTInfo.htds.add(htd.getHTableDescriptor());
1042         } catch (IOException ioe) {
1043           if (!orphanTableDirs.containsKey(tableName)) {
1044             LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1045             //should only report once for each table
1046             errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1047                 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1048             Set<String> columns = new HashSet<String>();
1049             orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1050           }
1051         }
1052       }
1053       if (!hbi.isSkipChecks()) {
1054         modTInfo.addRegionInfo(hbi);
1055       }
1056     }
1057 
1058     loadTableInfosForTablesWithNoRegion();
1059 
1060     return tablesInfo;
1061   }
1062 
1063   /**
1064    * To get the column family list according to the column family dirs
1065    * @param columns
1066    * @param hbi
1067    * @return a set of column families
1068    * @throws IOException
1069    */
1070   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
1071     Path regionDir = hbi.getHdfsRegionDir();
1072     FileSystem fs = regionDir.getFileSystem(getConf());
1073     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1074     for (FileStatus subdir : subDirs) {
1075       String columnfamily = subdir.getPath().getName();
1076       columns.add(columnfamily);
1077     }
1078     return columns;
1079   }
1080 
1081   /**
1082    * To fabricate a .tableinfo file with following contents<br>
1083    * 1. the correct tablename <br>
1084    * 2. the correct colfamily list<br>
1085    * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1086    * @throws IOException
1087    */
1088   private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1089       Set<String> columns) throws IOException {
1090     if (columns ==null || columns.isEmpty()) return false;
1091     HTableDescriptor htd = new HTableDescriptor(tableName);
1092     for (String columnfamimly : columns) {
1093       htd.addFamily(new HColumnDescriptor(columnfamimly));
1094     }
1095     fstd.createTableDescriptor(new TableDescriptor(htd, TableState.State.ENABLED), true);
1096     return true;
1097   }
1098 
1099   /**
1100    * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1101    * @throws IOException
1102    */
1103   public void fixEmptyMetaCells() throws IOException {
1104     if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1105       LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1106       for (Result region : emptyRegionInfoQualifiers) {
1107         deleteMetaRegion(region.getRow());
1108         errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1109       }
1110       emptyRegionInfoQualifiers.clear();
1111     }
1112   }
1113 
1114   /**
1115    * To fix orphan table by creating a .tableinfo file under tableDir <br>
1116    * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1117    * 2. else create a default .tableinfo file with following items<br>
1118    * &nbsp;2.1 the correct tablename <br>
1119    * &nbsp;2.2 the correct colfamily list<br>
1120    * &nbsp;2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1121    * @throws IOException
1122    */
1123   public void fixOrphanTables() throws IOException {
1124     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1125 
1126       List<TableName> tmpList = new ArrayList<TableName>();
1127       tmpList.addAll(orphanTableDirs.keySet());
1128       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1129       Iterator<Entry<TableName, Set<String>>> iter =
1130           orphanTableDirs.entrySet().iterator();
1131       int j = 0;
1132       int numFailedCase = 0;
1133       FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1134       while (iter.hasNext()) {
1135         Entry<TableName, Set<String>> entry =
1136             iter.next();
1137         TableName tableName = entry.getKey();
1138         LOG.info("Trying to fix orphan table error: " + tableName);
1139         if (j < htds.length) {
1140           if (tableName.equals(htds[j].getTableName())) {
1141             HTableDescriptor htd = htds[j];
1142             LOG.info("fixing orphan table: " + tableName + " from cache");
1143             fstd.createTableDescriptor(new TableDescriptor(htd, TableState.State.ENABLED), true);
1144             j++;
1145             iter.remove();
1146           }
1147         } else {
1148           if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1149             LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1150             LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1151             iter.remove();
1152           } else {
1153             LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1154             numFailedCase++;
1155           }
1156         }
1157         fixes++;
1158       }
1159 
1160       if (orphanTableDirs.isEmpty()) {
1161         // all orphanTableDirs are luckily recovered
1162         // re-run doFsck after recovering the .tableinfo file
1163         setShouldRerun();
1164         LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1165       } else if (numFailedCase > 0) {
1166         LOG.error("Failed to fix " + numFailedCase
1167             + " OrphanTables with default .tableinfo files");
1168       }
1169 
1170     }
1171     //cleanup the list
1172     orphanTableDirs.clear();
1173 
1174   }
1175 
1176   /**
1177    * This borrows code from MasterFileSystem.bootstrap()
1178    *
1179    * @return an open hbase:meta HRegion
1180    */
1181   private HRegion createNewMeta() throws IOException {
1182       Path rootdir = FSUtils.getRootDir(getConf());
1183     Configuration c = getConf();
1184     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1185     HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
1186     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false);
1187     HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor);
1188     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
1189     return meta;
1190   }
1191 
1192   /**
1193    * Generate set of puts to add to new meta.  This expects the tables to be
1194    * clean with no overlaps or holes.  If there are any problems it returns null.
1195    *
1196    * @return An array list of puts to do in bulk, null if tables have problems
1197    */
1198   private ArrayList<Put> generatePuts(
1199       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1200     ArrayList<Put> puts = new ArrayList<Put>();
1201     boolean hasProblems = false;
1202     for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1203       TableName name = e.getKey();
1204 
1205       // skip "hbase:meta"
1206       if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1207         continue;
1208       }
1209 
1210       TableInfo ti = e.getValue();
1211       for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1212           .entrySet()) {
1213         Collection<HbckInfo> his = spl.getValue();
1214         int sz = his.size();
1215         if (sz != 1) {
1216           // problem
1217           LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1218               + " had " +  sz + " regions instead of exactly 1." );
1219           hasProblems = true;
1220           continue;
1221         }
1222 
1223         // add the row directly to meta.
1224         HbckInfo hi = his.iterator().next();
1225         HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
1226         Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1227         puts.add(p);
1228       }
1229     }
1230     return hasProblems ? null : puts;
1231   }
1232 
1233   /**
1234    * Suggest fixes for each table
1235    */
1236   private void suggestFixes(
1237       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1238     for (TableInfo tInfo : tablesInfo.values()) {
1239       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1240       tInfo.checkRegionChain(handler);
1241     }
1242   }
1243 
1244   /**
1245    * Rebuilds meta from information in hdfs/fs.  Depends on configuration
1246    * settings passed into hbck constructor to point to a particular fs/dir.
1247    *
1248    * @param fix flag that determines if method should attempt to fix holes
1249    * @return true if successful, false if attempt failed.
1250    */
1251   public boolean rebuildMeta(boolean fix) throws IOException,
1252       InterruptedException {
1253 
1254     // TODO check to make sure hbase is offline. (or at least the table
1255     // currently being worked on is off line)
1256 
1257     // Determine what's on HDFS
1258     LOG.info("Loading HBase regioninfo from HDFS...");
1259     loadHdfsRegionDirs(); // populating regioninfo table.
1260 
1261     int errs = errors.getErrorList().size();
1262     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1263     checkHdfsIntegrity(false, false);
1264 
1265     // make sure ok.
1266     if (errors.getErrorList().size() != errs) {
1267       // While in error state, iterate until no more fixes possible
1268       while(true) {
1269         fixes = 0;
1270         suggestFixes(tablesInfo);
1271         errors.clear();
1272         loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1273         checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1274 
1275         int errCount = errors.getErrorList().size();
1276 
1277         if (fixes == 0) {
1278           if (errCount > 0) {
1279             return false; // failed to fix problems.
1280           } else {
1281             break; // no fixes and no problems? drop out and fix stuff!
1282           }
1283         }
1284       }
1285     }
1286 
1287     // we can rebuild, move old meta out of the way and start
1288     LOG.info("HDFS regioninfo's seems good.  Sidelining old hbase:meta");
1289     Path backupDir = sidelineOldMeta();
1290 
1291     LOG.info("Creating new hbase:meta");
1292     HRegion meta = createNewMeta();
1293 
1294     // populate meta
1295     List<Put> puts = generatePuts(tablesInfo);
1296     if (puts == null) {
1297       LOG.fatal("Problem encountered when creating new hbase:meta entries.  " +
1298         "You may need to restore the previously sidelined hbase:meta");
1299       return false;
1300     }
1301     meta.batchMutate(puts.toArray(new Put[puts.size()]));
1302     HRegion.closeHRegion(meta);
1303     LOG.info("Success! hbase:meta table rebuilt.");
1304     LOG.info("Old hbase:meta is moved into " + backupDir);
1305     return true;
1306   }
1307 
1308   private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1309       boolean fixOverlaps) throws IOException {
1310     LOG.info("Checking HBase region split map from HDFS data...");
1311     for (TableInfo tInfo : tablesInfo.values()) {
1312       TableIntegrityErrorHandler handler;
1313       if (fixHoles || fixOverlaps) {
1314         handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1315           fixHoles, fixOverlaps);
1316       } else {
1317         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1318       }
1319       if (!tInfo.checkRegionChain(handler)) {
1320         // should dump info as well.
1321         errors.report("Found inconsistency in table " + tInfo.getName());
1322       }
1323     }
1324     return tablesInfo;
1325   }
1326 
1327   private Path getSidelineDir() throws IOException {
1328     if (sidelineDir == null) {
1329       Path hbaseDir = FSUtils.getRootDir(getConf());
1330       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1331       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1332           + startMillis);
1333     }
1334     return sidelineDir;
1335   }
1336 
1337   /**
1338    * Sideline a region dir (instead of deleting it)
1339    */
1340   Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1341     return sidelineRegionDir(fs, null, hi);
1342   }
1343 
1344   /**
1345    * Sideline a region dir (instead of deleting it)
1346    *
1347    * @param parentDir if specified, the region will be sidelined to
1348    * folder like .../parentDir/<table name>/<region name>. The purpose
1349    * is to group together similar regions sidelined, for example, those
1350    * regions should be bulk loaded back later on. If null, it is ignored.
1351    */
1352   Path sidelineRegionDir(FileSystem fs,
1353       String parentDir, HbckInfo hi) throws IOException {
1354     TableName tableName = hi.getTableName();
1355     Path regionDir = hi.getHdfsRegionDir();
1356 
1357     if (!fs.exists(regionDir)) {
1358       LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1359       return null;
1360     }
1361 
1362     Path rootDir = getSidelineDir();
1363     if (parentDir != null) {
1364       rootDir = new Path(rootDir, parentDir);
1365     }
1366     Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1367     Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1368     fs.mkdirs(sidelineRegionDir);
1369     boolean success = false;
1370     FileStatus[] cfs =  fs.listStatus(regionDir);
1371     if (cfs == null) {
1372       LOG.info("Region dir is empty: " + regionDir);
1373     } else {
1374       for (FileStatus cf : cfs) {
1375         Path src = cf.getPath();
1376         Path dst =  new Path(sidelineRegionDir, src.getName());
1377         if (fs.isFile(src)) {
1378           // simple file
1379           success = fs.rename(src, dst);
1380           if (!success) {
1381             String msg = "Unable to rename file " + src +  " to " + dst;
1382             LOG.error(msg);
1383             throw new IOException(msg);
1384           }
1385           continue;
1386         }
1387 
1388         // is a directory.
1389         fs.mkdirs(dst);
1390 
1391         LOG.info("Sidelining files from " + src + " into containing region " + dst);
1392         // FileSystem.rename is inconsistent with directories -- if the
1393         // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1394         // it moves the src into the dst dir resulting in (foo/a/b).  If
1395         // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1396         FileStatus[] hfiles = fs.listStatus(src);
1397         if (hfiles != null && hfiles.length > 0) {
1398           for (FileStatus hfile : hfiles) {
1399             success = fs.rename(hfile.getPath(), dst);
1400             if (!success) {
1401               String msg = "Unable to rename file " + src +  " to " + dst;
1402               LOG.error(msg);
1403               throw new IOException(msg);
1404             }
1405           }
1406         }
1407         LOG.debug("Sideline directory contents:");
1408         debugLsr(sidelineRegionDir);
1409       }
1410     }
1411 
1412     LOG.info("Removing old region dir: " + regionDir);
1413     success = fs.delete(regionDir, true);
1414     if (!success) {
1415       String msg = "Unable to delete dir " + regionDir;
1416       LOG.error(msg);
1417       throw new IOException(msg);
1418     }
1419     return sidelineRegionDir;
1420   }
1421 
1422   /**
1423    * Side line an entire table.
1424    */
1425   void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1426       Path backupHbaseDir) throws IOException {
1427     Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1428     if (fs.exists(tableDir)) {
1429       Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1430       fs.mkdirs(backupTableDir.getParent());
1431       boolean success = fs.rename(tableDir, backupTableDir);
1432       if (!success) {
1433         throw new IOException("Failed to move  " + tableName + " from "
1434             +  tableDir + " to " + backupTableDir);
1435       }
1436     } else {
1437       LOG.info("No previous " + tableName +  " exists.  Continuing.");
1438     }
1439   }
1440 
1441   /**
1442    * @return Path to backup of original directory
1443    */
1444   Path sidelineOldMeta() throws IOException {
1445     // put current hbase:meta aside.
1446     Path hbaseDir = FSUtils.getRootDir(getConf());
1447     FileSystem fs = hbaseDir.getFileSystem(getConf());
1448     Path backupDir = getSidelineDir();
1449     fs.mkdirs(backupDir);
1450 
1451     try {
1452       sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1453     } catch (IOException e) {
1454         LOG.fatal("... failed to sideline meta. Currently in inconsistent state.  To restore "
1455             + "try to rename hbase:meta in " + backupDir.getName() + " to "
1456             + hbaseDir.getName() + ".", e);
1457       throw e; // throw original exception
1458     }
1459     return backupDir;
1460   }
1461 
1462   /**
1463    * Load the list of disabled tables in ZK into local set.
1464    * @throws ZooKeeperConnectionException
1465    * @throws IOException
1466    */
1467   private void loadDisabledTables()
1468   throws IOException {
1469     HConnectionManager.execute(new HConnectable<Void>(getConf()) {
1470       @Override
1471       public Void connect(HConnection connection) throws IOException {
1472         TableName[] tables = connection.listTableNames();
1473         for (TableName table : tables) {
1474           if (connection.getTableState(table)
1475               .inStates(TableState.State.DISABLED, TableState.State.DISABLING)) {
1476             disabledTables.add(table);
1477           }
1478         }
1479         return null;
1480       }
1481     });
1482   }
1483 
1484   /**
1485    * Check if the specified region's table is disabled.
1486    */
1487   private boolean isTableDisabled(HRegionInfo regionInfo) {
1488     return disabledTables.contains(regionInfo.getTable());
1489   }
1490 
1491   /**
1492    * Scan HDFS for all regions, recording their information into
1493    * regionInfoMap
1494    */
1495   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1496     Path rootDir = FSUtils.getRootDir(getConf());
1497     FileSystem fs = rootDir.getFileSystem(getConf());
1498 
1499     // list all tables from HDFS
1500     List<FileStatus> tableDirs = Lists.newArrayList();
1501 
1502     boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1503 
1504     List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1505     for (Path path : paths) {
1506       TableName tableName = FSUtils.getTableName(path);
1507        if ((!checkMetaOnly &&
1508            isTableIncluded(tableName)) ||
1509            tableName.equals(TableName.META_TABLE_NAME)) {
1510          tableDirs.add(fs.getFileStatus(path));
1511        }
1512     }
1513 
1514     // verify that version file exists
1515     if (!foundVersionFile) {
1516       errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1517           "Version file does not exist in root dir " + rootDir);
1518       if (shouldFixVersionFile()) {
1519         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1520             + " file.");
1521         setShouldRerun();
1522         FSUtils.setVersion(fs, rootDir, getConf().getInt(
1523             HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1524             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1525             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1526       }
1527     }
1528 
1529     // level 1:  <HBASE_DIR>/*
1530     List<WorkItemHdfsDir> dirs = new ArrayList<WorkItemHdfsDir>(tableDirs.size());
1531     List<Future<Void>> dirsFutures;
1532 
1533     for (FileStatus tableDir : tableDirs) {
1534       LOG.debug("Loading region dirs from " +tableDir.getPath());
1535       dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir));
1536     }
1537 
1538     // Invoke and wait for Callables to complete
1539     dirsFutures = executor.invokeAll(dirs);
1540 
1541     for(Future<Void> f: dirsFutures) {
1542       try {
1543         f.get();
1544       } catch(ExecutionException e) {
1545         LOG.warn("Could not load region dir " , e.getCause());
1546       }
1547     }
1548   }
1549 
1550   /**
1551    * Record the location of the hbase:meta region as found in ZooKeeper.
1552    */
1553   private boolean recordMetaRegion() throws IOException {
1554     HRegionLocation metaLocation = connection.locateRegion(
1555       TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW);
1556 
1557     // Check if Meta region is valid and existing
1558     if (metaLocation == null || metaLocation.getRegionInfo() == null ||
1559         metaLocation.getHostname() == null) {
1560       errors.reportError(ERROR_CODE.NULL_META_REGION,
1561         "META region or some of its attributes are null.");
1562       return false;
1563     }
1564     ServerName sn;
1565     try {
1566       sn = getMetaRegionServerName();
1567     } catch (KeeperException e) {
1568       throw new IOException(e);
1569     }
1570     MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, System.currentTimeMillis());
1571     HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1572     if (hbckInfo == null) {
1573       regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1574     } else {
1575       hbckInfo.metaEntry = m;
1576     }
1577     return true;
1578   }
1579 
1580   private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1581     return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1582       @Override
1583       public void abort(String why, Throwable e) {
1584         LOG.error(why, e);
1585         System.exit(1);
1586       }
1587 
1588       @Override
1589       public boolean isAborted() {
1590         return false;
1591       }
1592 
1593     });
1594   }
1595 
1596   private ServerName getMetaRegionServerName()
1597   throws IOException, KeeperException {
1598     ZooKeeperWatcher zkw = createZooKeeperWatcher();
1599     ServerName sn = null;
1600     try {
1601       sn = new MetaTableLocator().getMetaRegionLocation(zkw);
1602     } finally {
1603       zkw.close();
1604     }
1605     return sn;
1606   }
1607 
1608   /**
1609    * Contacts each regionserver and fetches metadata about regions.
1610    * @param regionServerList - the list of region servers to connect to
1611    * @throws IOException if a remote or network exception occurs
1612    */
1613   void processRegionServers(Collection<ServerName> regionServerList)
1614     throws IOException, InterruptedException {
1615 
1616     List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1617     List<Future<Void>> workFutures;
1618 
1619     // loop to contact each region server in parallel
1620     for (ServerName rsinfo: regionServerList) {
1621       workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1622     }
1623 
1624     workFutures = executor.invokeAll(workItems);
1625 
1626     for(int i=0; i<workFutures.size(); i++) {
1627       WorkItemRegion item = workItems.get(i);
1628       Future<Void> f = workFutures.get(i);
1629       try {
1630         f.get();
1631       } catch(ExecutionException e) {
1632         LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1633             e.getCause());
1634       }
1635     }
1636   }
1637 
1638   /**
1639    * Check consistency of all regions that have been found in previous phases.
1640    */
1641   private void checkAndFixConsistency()
1642   throws IOException, KeeperException, InterruptedException {
1643 	  // Divide the checks in two phases. One for default/primary replicas and another
1644 	  // for the non-primary ones. Keeps code cleaner this way.
1645     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1646       if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1647         checkRegionConsistency(e.getKey(), e.getValue());
1648       }
1649     }
1650     boolean prevHdfsCheck = shouldCheckHdfs();
1651     setCheckHdfs(false); //replicas don't have any hdfs data
1652     // Run a pass over the replicas and fix any assignment issues that exist on the currently
1653     // deployed/undeployed replicas.
1654     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1655       if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1656         checkRegionConsistency(e.getKey(), e.getValue());
1657       }
1658     }
1659     setCheckHdfs(prevHdfsCheck);
1660   }
1661 
1662   private void preCheckPermission() throws IOException, AccessDeniedException {
1663     if (shouldIgnorePreCheckPermission()) {
1664       return;
1665     }
1666 
1667     Path hbaseDir = FSUtils.getRootDir(getConf());
1668     FileSystem fs = hbaseDir.getFileSystem(getConf());
1669     UserProvider userProvider = UserProvider.instantiate(getConf());
1670     UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1671     FileStatus[] files = fs.listStatus(hbaseDir);
1672     for (FileStatus file : files) {
1673       try {
1674         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1675       } catch (AccessDeniedException ace) {
1676         LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1677         errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1678           + " does not have write perms to " + file.getPath()
1679           + ". Please rerun hbck as hdfs user " + file.getOwner());
1680         throw ace;
1681       }
1682     }
1683   }
1684 
1685   /**
1686    * Deletes region from meta table
1687    */
1688   private void deleteMetaRegion(HbckInfo hi) throws IOException {
1689     deleteMetaRegion(hi.metaEntry.getRegionName());
1690   }
1691 
1692   /**
1693    * Deletes region from meta table
1694    */
1695   private void deleteMetaRegion(byte[] metaKey) throws IOException {
1696     Delete d = new Delete(metaKey);
1697     meta.delete(d);
1698     meta.flushCommits();
1699     LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1700   }
1701 
1702   /**
1703    * Reset the split parent region info in meta table
1704    */
1705   private void resetSplitParent(HbckInfo hi) throws IOException {
1706     RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
1707     Delete d = new Delete(hi.metaEntry.getRegionName());
1708     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1709     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1710     mutations.add(d);
1711 
1712     HRegionInfo hri = new HRegionInfo(hi.metaEntry);
1713     hri.setOffline(false);
1714     hri.setSplit(false);
1715     Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1716     mutations.add(p);
1717 
1718     meta.mutateRow(mutations);
1719     meta.flushCommits();
1720     LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
1721   }
1722 
1723   /**
1724    * This backwards-compatibility wrapper for permanently offlining a region
1725    * that should not be alive.  If the region server does not support the
1726    * "offline" method, it will use the closest unassign method instead.  This
1727    * will basically work until one attempts to disable or delete the affected
1728    * table.  The problem has to do with in-memory only master state, so
1729    * restarting the HMaster or failing over to another should fix this.
1730    */
1731   private void offline(byte[] regionName) throws IOException {
1732     String regionString = Bytes.toStringBinary(regionName);
1733     if (!rsSupportsOffline) {
1734       LOG.warn("Using unassign region " + regionString
1735           + " instead of using offline method, you should"
1736           + " restart HMaster after these repairs");
1737       admin.unassign(regionName, true);
1738       return;
1739     }
1740 
1741     // first time we assume the rs's supports #offline.
1742     try {
1743       LOG.info("Offlining region " + regionString);
1744       admin.offline(regionName);
1745     } catch (IOException ioe) {
1746       String notFoundMsg = "java.lang.NoSuchMethodException: " +
1747         "org.apache.hadoop.hbase.master.HMaster.offline([B)";
1748       if (ioe.getMessage().contains(notFoundMsg)) {
1749         LOG.warn("Using unassign region " + regionString
1750             + " instead of using offline method, you should"
1751             + " restart HMaster after these repairs");
1752         rsSupportsOffline = false; // in the future just use unassign
1753         admin.unassign(regionName, true);
1754         return;
1755       }
1756       throw ioe;
1757     }
1758   }
1759 
1760   private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
1761     undeployRegionsForHbi(hi);
1762     // undeploy replicas of the region (but only if the method is invoked for the primary)
1763     if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1764       return;
1765     }
1766     int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
1767     for (int i = 1; i < numReplicas; i++) {
1768       if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
1769       HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
1770           hi.getPrimaryHRIForDeployedReplica(), i);
1771       HbckInfo h = regionInfoMap.get(hri.getEncodedName());
1772       if (h != null) {
1773         undeployRegionsForHbi(h);
1774         //set skip checks; we undeployed it, and we don't want to evaluate this anymore
1775         //in consistency checks
1776         h.setSkipChecks(true);
1777       }
1778     }
1779   }
1780 
1781   private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
1782     for (OnlineEntry rse : hi.deployedEntries) {
1783       LOG.debug("Undeploy region "  + rse.hri + " from " + rse.hsa);
1784       try {
1785         HBaseFsckRepair.closeRegionSilentlyAndWait(admin, rse.hsa, rse.hri);
1786         offline(rse.hri.getRegionName());
1787       } catch (IOException ioe) {
1788         LOG.warn("Got exception when attempting to offline region "
1789             + Bytes.toString(rse.hri.getRegionName()), ioe);
1790       }
1791     }
1792   }
1793 
1794   /**
1795    * Attempts to undeploy a region from a region server based in information in
1796    * META.  Any operations that modify the file system should make sure that
1797    * its corresponding region is not deployed to prevent data races.
1798    *
1799    * A separate call is required to update the master in-memory region state
1800    * kept in the AssignementManager.  Because disable uses this state instead of
1801    * that found in META, we can't seem to cleanly disable/delete tables that
1802    * have been hbck fixed.  When used on a version of HBase that does not have
1803    * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master
1804    * restart or failover may be required.
1805    */
1806   private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
1807     if (hi.metaEntry == null && hi.hdfsEntry == null) {
1808       undeployRegions(hi);
1809       return;
1810     }
1811 
1812     // get assignment info and hregioninfo from meta.
1813     Get get = new Get(hi.getRegionName());
1814     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1815     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
1816     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
1817     // also get the locations of the replicas to close if the primary region is being closed
1818     if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1819       int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
1820       for (int i = 0; i < numReplicas; i++) {
1821         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
1822         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
1823       }
1824     }
1825     Result r = meta.get(get);
1826     RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
1827     if (rl == null) {
1828       LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
1829           " since meta does not have handle to reach it");
1830       return;
1831     }
1832     for (HRegionLocation h : rl.getRegionLocations()) {
1833       ServerName serverName = h.getServerName();
1834       if (serverName == null) {
1835         errors.reportError("Unable to close region "
1836             + hi.getRegionNameAsString() +  " because meta does not "
1837             + "have handle to reach it.");
1838         continue;
1839       }
1840       HRegionInfo hri = h.getRegionInfo();
1841       if (hri == null) {
1842         LOG.warn("Unable to close region " + hi.getRegionNameAsString()
1843             + " because hbase:meta had invalid or missing "
1844             + HConstants.CATALOG_FAMILY_STR + ":"
1845             + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
1846             + " qualifier value.");
1847         continue;
1848       }
1849       // close the region -- close files and remove assignment
1850       HBaseFsckRepair.closeRegionSilentlyAndWait(admin, serverName, hri);
1851     }
1852   }
1853 
1854   private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
1855     KeeperException, InterruptedException {
1856     // If we are trying to fix the errors
1857     if (shouldFixAssignments()) {
1858       errors.print(msg);
1859       undeployRegions(hbi);
1860       setShouldRerun();
1861       HRegionInfo hri = hbi.getHdfsHRI();
1862       if (hri == null) {
1863         hri = hbi.metaEntry;
1864       }
1865       HBaseFsckRepair.fixUnassigned(admin, hri);
1866       HBaseFsckRepair.waitUntilAssigned(admin, hri);
1867 
1868       // also assign replicas if needed (do it only when this call operates on a primary replica)
1869       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
1870       int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
1871       for (int i = 1; i < replicationCount; i++) {
1872         hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
1873         HbckInfo h = regionInfoMap.get(hri.getEncodedName());
1874         if (h != null) {
1875           undeployRegions(h);
1876           //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore
1877           //in consistency checks
1878           h.setSkipChecks(true);
1879         }
1880         HBaseFsckRepair.fixUnassigned(admin, hri);
1881         HBaseFsckRepair.waitUntilAssigned(admin, hri);
1882       }
1883 
1884     }
1885   }
1886 
1887   /**
1888    * Check a single region for consistency and correct deployment.
1889    */
1890   private void checkRegionConsistency(final String key, final HbckInfo hbi)
1891   throws IOException, KeeperException, InterruptedException {
1892 
1893 	if (hbi.isSkipChecks()) return;
1894 	String descriptiveName = hbi.toString();
1895     boolean inMeta = hbi.metaEntry != null;
1896     // In case not checking HDFS, assume the region is on HDFS
1897     boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
1898     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
1899     boolean isDeployed = !hbi.deployedOn.isEmpty();
1900     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
1901     boolean deploymentMatchesMeta =
1902       hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
1903       hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
1904     boolean splitParent =
1905       (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
1906     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
1907     boolean recentlyModified = inHdfs &&
1908       hbi.getModTime() + timelag > System.currentTimeMillis();
1909 
1910     // ========== First the healthy cases =============
1911     if (hbi.containsOnlyHdfsEdits()) {
1912       return;
1913     }
1914     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
1915       return;
1916     } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
1917       LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
1918         "tabled that is not deployed");
1919       return;
1920     } else if (recentlyModified) {
1921       LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
1922       return;
1923     }
1924     // ========== Cases where the region is not in hbase:meta =============
1925     else if (!inMeta && !inHdfs && !isDeployed) {
1926       // We shouldn't have record of this region at all then!
1927       assert false : "Entry for region with no data";
1928     } else if (!inMeta && !inHdfs && isDeployed) {
1929       errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
1930           + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
1931           "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1932       if (shouldFixAssignments()) {
1933         undeployRegions(hbi);
1934       }
1935 
1936     } else if (!inMeta && inHdfs && !isDeployed) {
1937       if (hbi.isMerged()) {
1938         // This region has already been merged, the remaining hdfs file will be
1939         // cleaned by CatalogJanitor later
1940         hbi.setSkipChecks(true);
1941         LOG.info("Region " + descriptiveName
1942             + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
1943         return;
1944       }
1945       errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
1946           + descriptiveName + " on HDFS, but not listed in hbase:meta " +
1947           "or deployed on any region server");
1948       // restore region consistency of an adopted orphan
1949       if (shouldFixMeta()) {
1950         if (!hbi.isHdfsRegioninfoPresent()) {
1951           LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
1952               +  " in table integrity repair phase if -fixHdfsOrphans was" +
1953               " used.");
1954           return;
1955         }
1956 
1957         LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
1958         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
1959         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
1960             admin.getClusterStatus().getServers(), numReplicas);
1961 
1962         tryAssignmentRepair(hbi, "Trying to reassign region...");
1963       }
1964 
1965     } else if (!inMeta && inHdfs && isDeployed) {
1966       errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
1967           + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1968       debugLsr(hbi.getHdfsRegionDir());
1969       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1970         // for replicas, this means that we should undeploy the region (we would have
1971         // gone over the primaries and fixed meta holes in first phase under
1972         // checkAndFixConsistency; we shouldn't get the condition !inMeta at
1973         // this stage unless unwanted replica)
1974         if (shouldFixAssignments()) {
1975           undeployRegionsForHbi(hbi);
1976         }
1977       }
1978       if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1979         if (!hbi.isHdfsRegioninfoPresent()) {
1980           LOG.error("This should have been repaired in table integrity repair phase");
1981           return;
1982         }
1983 
1984         LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
1985         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
1986         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
1987             admin.getClusterStatus().getServers(), numReplicas);
1988         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
1989       }
1990 
1991     // ========== Cases where the region is in hbase:meta =============
1992     } else if (inMeta && inHdfs && !isDeployed && splitParent) {
1993       // check whether this is an actual error, or just transient state where parent
1994       // is not cleaned
1995       if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
1996         // check that split daughters are there
1997         HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
1998         HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
1999         if (infoA != null && infoB != null) {
2000           // we already processed or will process daughters. Move on, nothing to see here.
2001           hbi.setSkipChecks(true);
2002           return;
2003         }
2004       }
2005       errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2006           + descriptiveName + " is a split parent in META, in HDFS, "
2007           + "and not deployed on any region server. This could be transient.");
2008       if (shouldFixSplitParents()) {
2009         setShouldRerun();
2010         resetSplitParent(hbi);
2011       }
2012     } else if (inMeta && !inHdfs && !isDeployed) {
2013       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2014           + descriptiveName + " found in META, but not in HDFS "
2015           + "or deployed on any region server.");
2016       if (shouldFixMeta()) {
2017         deleteMetaRegion(hbi);
2018       }
2019     } else if (inMeta && !inHdfs && isDeployed) {
2020       errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2021           + " found in META, but not in HDFS, " +
2022           "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2023       // We treat HDFS as ground truth.  Any information in meta is transient
2024       // and equivalent data can be regenerated.  So, lets unassign and remove
2025       // these problems from META.
2026       if (shouldFixAssignments()) {
2027         errors.print("Trying to fix unassigned region...");
2028         undeployRegions(hbi);
2029       }
2030       if (shouldFixMeta()) {
2031         // wait for it to complete
2032         deleteMetaRegion(hbi);
2033       }
2034     } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2035       errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2036           + " not deployed on any region server.");
2037       tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2038     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2039       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2040           "Region " + descriptiveName + " should not be deployed according " +
2041           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2042       if (shouldFixAssignments()) {
2043         errors.print("Trying to close the region " + descriptiveName);
2044         setShouldRerun();
2045         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
2046       }
2047     } else if (inMeta && inHdfs && isMultiplyDeployed) {
2048       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2049           + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
2050           + " but is multiply assigned to region servers " +
2051           Joiner.on(", ").join(hbi.deployedOn));
2052       // If we are trying to fix the errors
2053       if (shouldFixAssignments()) {
2054         errors.print("Trying to fix assignment error...");
2055         setShouldRerun();
2056         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
2057       }
2058     } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2059       errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2060           + descriptiveName + " listed in hbase:meta on region server " +
2061           hbi.metaEntry.regionServer + " but found on region server " +
2062           hbi.deployedOn.get(0));
2063       // If we are trying to fix the errors
2064       if (shouldFixAssignments()) {
2065         errors.print("Trying to fix assignment error...");
2066         setShouldRerun();
2067         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
2068         HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2069       }
2070     } else {
2071       errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2072           " is in an unforeseen state:" +
2073           " inMeta=" + inMeta +
2074           " inHdfs=" + inHdfs +
2075           " isDeployed=" + isDeployed +
2076           " isMultiplyDeployed=" + isMultiplyDeployed +
2077           " deploymentMatchesMeta=" + deploymentMatchesMeta +
2078           " shouldBeDeployed=" + shouldBeDeployed);
2079     }
2080   }
2081 
2082   /**
2083    * Checks tables integrity. Goes over all regions and scans the tables.
2084    * Collects all the pieces for each table and checks if there are missing,
2085    * repeated or overlapping ones.
2086    * @throws IOException
2087    */
2088   SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2089     tablesInfo = new TreeMap<TableName,TableInfo> ();
2090     LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2091     for (HbckInfo hbi : regionInfoMap.values()) {
2092       // Check only valid, working regions
2093       if (hbi.metaEntry == null) {
2094         // this assumes that consistency check has run loadMetaEntry
2095         Path p = hbi.getHdfsRegionDir();
2096         if (p == null) {
2097           errors.report("No regioninfo in Meta or HDFS. " + hbi);
2098         }
2099 
2100         // TODO test.
2101         continue;
2102       }
2103       if (hbi.metaEntry.regionServer == null) {
2104         errors.detail("Skipping region because no region server: " + hbi);
2105         continue;
2106       }
2107       if (hbi.metaEntry.isOffline()) {
2108         errors.detail("Skipping region because it is offline: " + hbi);
2109         continue;
2110       }
2111       if (hbi.containsOnlyHdfsEdits()) {
2112         errors.detail("Skipping region because it only contains edits" + hbi);
2113         continue;
2114       }
2115 
2116       // Missing regionDir or over-deployment is checked elsewhere. Include
2117       // these cases in modTInfo, so we can evaluate those regions as part of
2118       // the region chain in META
2119       //if (hbi.foundRegionDir == null) continue;
2120       //if (hbi.deployedOn.size() != 1) continue;
2121       if (hbi.deployedOn.size() == 0) continue;
2122 
2123       // We should be safe here
2124       TableName tableName = hbi.metaEntry.getTable();
2125       TableInfo modTInfo = tablesInfo.get(tableName);
2126       if (modTInfo == null) {
2127         modTInfo = new TableInfo(tableName);
2128       }
2129       for (ServerName server : hbi.deployedOn) {
2130         modTInfo.addServer(server);
2131       }
2132 
2133       if (!hbi.isSkipChecks()) {
2134         modTInfo.addRegionInfo(hbi);
2135       }
2136 
2137       tablesInfo.put(tableName, modTInfo);
2138     }
2139 
2140     loadTableInfosForTablesWithNoRegion();
2141 
2142     for (TableInfo tInfo : tablesInfo.values()) {
2143       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2144       if (!tInfo.checkRegionChain(handler)) {
2145         errors.report("Found inconsistency in table " + tInfo.getName());
2146       }
2147     }
2148     return tablesInfo;
2149   }
2150 
2151   /** Loads table info's for tables that may not have been included, since there are no
2152    * regions reported for the table, but table dir is there in hdfs
2153    */
2154   private void loadTableInfosForTablesWithNoRegion() throws IOException {
2155     Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2156     for (HTableDescriptor htd : allTables.values()) {
2157       if (checkMetaOnly && !htd.isMetaTable()) {
2158         continue;
2159       }
2160 
2161       TableName tableName = htd.getTableName();
2162       if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2163         TableInfo tableInfo = new TableInfo(tableName);
2164         tableInfo.htds.add(htd);
2165         tablesInfo.put(htd.getTableName(), tableInfo);
2166       }
2167     }
2168   }
2169 
2170   /**
2171    * Merge hdfs data by moving from contained HbckInfo into targetRegionDir.
2172    * @return number of file move fixes done to merge regions.
2173    */
2174   public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2175     int fileMoves = 0;
2176     String thread = Thread.currentThread().getName();
2177     LOG.debug("[" + thread + "] Contained region dir after close and pause");
2178     debugLsr(contained.getHdfsRegionDir());
2179 
2180     // rename the contained into the container.
2181     FileSystem fs = targetRegionDir.getFileSystem(getConf());
2182     FileStatus[] dirs = null;
2183     try {
2184       dirs = fs.listStatus(contained.getHdfsRegionDir());
2185     } catch (FileNotFoundException fnfe) {
2186       // region we are attempting to merge in is not present!  Since this is a merge, there is
2187       // no harm skipping this region if it does not exist.
2188       if (!fs.exists(contained.getHdfsRegionDir())) {
2189         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2190             + " is missing. Assuming already sidelined or moved.");
2191       } else {
2192         sidelineRegionDir(fs, contained);
2193       }
2194       return fileMoves;
2195     }
2196 
2197     if (dirs == null) {
2198       if (!fs.exists(contained.getHdfsRegionDir())) {
2199         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2200             + " already sidelined.");
2201       } else {
2202         sidelineRegionDir(fs, contained);
2203       }
2204       return fileMoves;
2205     }
2206 
2207     for (FileStatus cf : dirs) {
2208       Path src = cf.getPath();
2209       Path dst =  new Path(targetRegionDir, src.getName());
2210 
2211       if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2212         // do not copy the old .regioninfo file.
2213         continue;
2214       }
2215 
2216       if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2217         // do not copy the .oldlogs files
2218         continue;
2219       }
2220 
2221       LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2222       // FileSystem.rename is inconsistent with directories -- if the
2223       // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2224       // it moves the src into the dst dir resulting in (foo/a/b).  If
2225       // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2226       for (FileStatus hfile : fs.listStatus(src)) {
2227         boolean success = fs.rename(hfile.getPath(), dst);
2228         if (success) {
2229           fileMoves++;
2230         }
2231       }
2232       LOG.debug("[" + thread + "] Sideline directory contents:");
2233       debugLsr(targetRegionDir);
2234     }
2235 
2236     // if all success.
2237     sidelineRegionDir(fs, contained);
2238     LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2239         getSidelineDir());
2240     debugLsr(contained.getHdfsRegionDir());
2241 
2242     return fileMoves;
2243   }
2244 
2245 
2246   static class WorkItemOverlapMerge implements Callable<Void> {
2247     private TableIntegrityErrorHandler handler;
2248     Collection<HbckInfo> overlapgroup;
2249 
2250     WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2251       this.handler = handler;
2252       this.overlapgroup = overlapgroup;
2253     }
2254 
2255     @Override
2256     public Void call() throws Exception {
2257       handler.handleOverlapGroup(overlapgroup);
2258       return null;
2259     }
2260   };
2261 
2262 
2263   /**
2264    * Maintain information about a particular table.
2265    */
2266   public class TableInfo {
2267     TableName tableName;
2268     TreeSet <ServerName> deployedOn;
2269 
2270     // backwards regions
2271     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2272 
2273     // sidelined big overlapped regions
2274     final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2275 
2276     // region split calculator
2277     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2278 
2279     // Histogram of different HTableDescriptors found.  Ideally there is only one!
2280     final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2281 
2282     // key = start split, values = set of splits in problem group
2283     final Multimap<byte[], HbckInfo> overlapGroups =
2284       TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2285 
2286     TableInfo(TableName name) {
2287       this.tableName = name;
2288       deployedOn = new TreeSet <ServerName>();
2289     }
2290 
2291     /**
2292      * @return descriptor common to all regions.  null if are none or multiple!
2293      */
2294     private HTableDescriptor getHTD() {
2295       if (htds.size() == 1) {
2296         return (HTableDescriptor)htds.toArray()[0];
2297       } else {
2298         LOG.error("None/Multiple table descriptors found for table '"
2299           + tableName + "' regions: " + htds);
2300       }
2301       return null;
2302     }
2303 
2304     public void addRegionInfo(HbckInfo hir) {
2305       if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2306         // end key is absolute end key, just add it.
2307         // ignore replicas other than primary for these checks
2308         if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2309         return;
2310       }
2311 
2312       // if not the absolute end key, check for cycle
2313       if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2314         errors.reportError(
2315             ERROR_CODE.REGION_CYCLE,
2316             String.format("The endkey for this region comes before the "
2317                 + "startkey, startkey=%s, endkey=%s",
2318                 Bytes.toStringBinary(hir.getStartKey()),
2319                 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2320         backwards.add(hir);
2321         return;
2322       }
2323 
2324       // main case, add to split calculator
2325       // ignore replicas other than primary for these checks
2326       if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2327     }
2328 
2329     public void addServer(ServerName server) {
2330       this.deployedOn.add(server);
2331     }
2332 
2333     public TableName getName() {
2334       return tableName;
2335     }
2336 
2337     public int getNumRegions() {
2338       return sc.getStarts().size() + backwards.size();
2339     }
2340 
2341     private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2342       ErrorReporter errors;
2343 
2344       IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2345         this.errors = errors;
2346         setTableInfo(ti);
2347       }
2348 
2349       @Override
2350       public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2351         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2352             "First region should start with an empty key.  You need to "
2353             + " create a new region and regioninfo in HDFS to plug the hole.",
2354             getTableInfo(), hi);
2355       }
2356 
2357       @Override
2358       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2359         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2360             "Last region should end with an empty key. You need to "
2361                 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2362       }
2363 
2364       @Override
2365       public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2366         errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2367             "Region has the same start and end key.", getTableInfo(), hi);
2368       }
2369 
2370       @Override
2371       public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2372         byte[] key = r1.getStartKey();
2373         // dup start key
2374         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2375             "Multiple regions have the same startkey: "
2376             + Bytes.toStringBinary(key), getTableInfo(), r1);
2377         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2378             "Multiple regions have the same startkey: "
2379             + Bytes.toStringBinary(key), getTableInfo(), r2);
2380       }
2381 
2382       @Override
2383       public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2384         errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2385             "There is an overlap in the region chain.",
2386             getTableInfo(), hi1, hi2);
2387       }
2388 
2389       @Override
2390       public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2391         errors.reportError(
2392             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2393             "There is a hole in the region chain between "
2394                 + Bytes.toStringBinary(holeStart) + " and "
2395                 + Bytes.toStringBinary(holeStop)
2396                 + ".  You need to create a new .regioninfo and region "
2397                 + "dir in hdfs to plug the hole.");
2398       }
2399     };
2400 
2401     /**
2402      * This handler fixes integrity errors from hdfs information.  There are
2403      * basically three classes of integrity problems 1) holes, 2) overlaps, and
2404      * 3) invalid regions.
2405      *
2406      * This class overrides methods that fix holes and the overlap group case.
2407      * Individual cases of particular overlaps are handled by the general
2408      * overlap group merge repair case.
2409      *
2410      * If hbase is online, this forces regions offline before doing merge
2411      * operations.
2412      */
2413     private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2414       Configuration conf;
2415 
2416       boolean fixOverlaps = true;
2417 
2418       HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2419           boolean fixHoles, boolean fixOverlaps) {
2420         super(ti, errors);
2421         this.conf = conf;
2422         this.fixOverlaps = fixOverlaps;
2423         // TODO properly use fixHoles
2424       }
2425 
2426       /**
2427        * This is a special case hole -- when the first region of a table is
2428        * missing from META, HBase doesn't acknowledge the existance of the
2429        * table.
2430        */
2431       @Override
2432       public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2433         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2434             "First region should start with an empty key.  Creating a new " +
2435             "region and regioninfo in HDFS to plug the hole.",
2436             getTableInfo(), next);
2437         HTableDescriptor htd = getTableInfo().getHTD();
2438         // from special EMPTY_START_ROW to next region's startKey
2439         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2440             HConstants.EMPTY_START_ROW, next.getStartKey());
2441 
2442         // TODO test
2443         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2444         LOG.info("Table region start key was not empty.  Created new empty region: "
2445             + newRegion + " " +region);
2446         fixes++;
2447       }
2448 
2449       @Override
2450       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2451         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2452             "Last region should end with an empty key.  Creating a new "
2453                 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2454         HTableDescriptor htd = getTableInfo().getHTD();
2455         // from curEndKey to EMPTY_START_ROW
2456         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2457             HConstants.EMPTY_START_ROW);
2458 
2459         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2460         LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
2461             + " " + region);
2462         fixes++;
2463       }
2464 
2465       /**
2466        * There is a hole in the hdfs regions that violates the table integrity
2467        * rules.  Create a new empty region that patches the hole.
2468        */
2469       @Override
2470       public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2471         errors.reportError(
2472             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2473             "There is a hole in the region chain between "
2474                 + Bytes.toStringBinary(holeStartKey) + " and "
2475                 + Bytes.toStringBinary(holeStopKey)
2476                 + ".  Creating a new regioninfo and region "
2477                 + "dir in hdfs to plug the hole.");
2478         HTableDescriptor htd = getTableInfo().getHTD();
2479         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2480         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2481         LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2482         fixes++;
2483       }
2484 
2485       /**
2486        * This takes set of overlapping regions and merges them into a single
2487        * region.  This covers cases like degenerate regions, shared start key,
2488        * general overlaps, duplicate ranges, and partial overlapping regions.
2489        *
2490        * Cases:
2491        * - Clean regions that overlap
2492        * - Only .oldlogs regions (can't find start/stop range, or figure out)
2493        *
2494        * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
2495        */
2496       @Override
2497       public void handleOverlapGroup(Collection<HbckInfo> overlap)
2498           throws IOException {
2499         Preconditions.checkNotNull(overlap);
2500         Preconditions.checkArgument(overlap.size() >0);
2501 
2502         if (!this.fixOverlaps) {
2503           LOG.warn("Not attempting to repair overlaps.");
2504           return;
2505         }
2506 
2507         if (overlap.size() > maxMerge) {
2508           LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2509             "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2510           if (sidelineBigOverlaps) {
2511             // we only sideline big overlapped groups that exceeds the max number of regions to merge
2512             sidelineBigOverlaps(overlap);
2513           }
2514           return;
2515         }
2516 
2517         mergeOverlaps(overlap);
2518       }
2519 
2520       void mergeOverlaps(Collection<HbckInfo> overlap)
2521           throws IOException {
2522         String thread = Thread.currentThread().getName();
2523         LOG.info("== [" + thread + "] Merging regions into one region: "
2524           + Joiner.on(",").join(overlap));
2525         // get the min / max range and close all concerned regions
2526         Pair<byte[], byte[]> range = null;
2527         for (HbckInfo hi : overlap) {
2528           if (range == null) {
2529             range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2530           } else {
2531             if (RegionSplitCalculator.BYTES_COMPARATOR
2532                 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2533               range.setFirst(hi.getStartKey());
2534             }
2535             if (RegionSplitCalculator.BYTES_COMPARATOR
2536                 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2537               range.setSecond(hi.getEndKey());
2538             }
2539           }
2540           // need to close files so delete can happen.
2541           LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
2542           LOG.debug("[" + thread + "] Contained region dir before close");
2543           debugLsr(hi.getHdfsRegionDir());
2544           try {
2545             LOG.info("[" + thread + "] Closing region: " + hi);
2546             closeRegion(hi);
2547           } catch (IOException ioe) {
2548             LOG.warn("[" + thread + "] Was unable to close region " + hi
2549               + ".  Just continuing... ", ioe);
2550           } catch (InterruptedException e) {
2551             LOG.warn("[" + thread + "] Was unable to close region " + hi
2552               + ".  Just continuing... ", e);
2553           }
2554 
2555           try {
2556             LOG.info("[" + thread + "] Offlining region: " + hi);
2557             offline(hi.getRegionName());
2558           } catch (IOException ioe) {
2559             LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2560               + ".  Just continuing... ", ioe);
2561           }
2562         }
2563 
2564         // create new empty container region.
2565         HTableDescriptor htd = getTableInfo().getHTD();
2566         // from start key to end Key
2567         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
2568             range.getSecond());
2569         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2570         LOG.info("[" + thread + "] Created new empty container region: " +
2571             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
2572         debugLsr(region.getRegionFileSystem().getRegionDir());
2573 
2574         // all target regions are closed, should be able to safely cleanup.
2575         boolean didFix= false;
2576         Path target = region.getRegionFileSystem().getRegionDir();
2577         for (HbckInfo contained : overlap) {
2578           LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
2579           int merges = mergeRegionDirs(target, contained);
2580           if (merges > 0) {
2581             didFix = true;
2582           }
2583         }
2584         if (didFix) {
2585           fixes++;
2586         }
2587       }
2588 
2589       /**
2590        * Sideline some regions in a big overlap group so that it
2591        * will have fewer regions, and it is easier to merge them later on.
2592        *
2593        * @param bigOverlap the overlapped group with regions more than maxMerge
2594        * @throws IOException
2595        */
2596       void sidelineBigOverlaps(
2597           Collection<HbckInfo> bigOverlap) throws IOException {
2598         int overlapsToSideline = bigOverlap.size() - maxMerge;
2599         if (overlapsToSideline > maxOverlapsToSideline) {
2600           overlapsToSideline = maxOverlapsToSideline;
2601         }
2602         List<HbckInfo> regionsToSideline =
2603           RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
2604         FileSystem fs = FileSystem.get(conf);
2605         for (HbckInfo regionToSideline: regionsToSideline) {
2606           try {
2607             LOG.info("Closing region: " + regionToSideline);
2608             closeRegion(regionToSideline);
2609           } catch (IOException ioe) {
2610             LOG.warn("Was unable to close region " + regionToSideline
2611               + ".  Just continuing... ", ioe);
2612           } catch (InterruptedException e) {
2613             LOG.warn("Was unable to close region " + regionToSideline
2614               + ".  Just continuing... ", e);
2615           }
2616 
2617           try {
2618             LOG.info("Offlining region: " + regionToSideline);
2619             offline(regionToSideline.getRegionName());
2620           } catch (IOException ioe) {
2621             LOG.warn("Unable to offline region from master: " + regionToSideline
2622               + ".  Just continuing... ", ioe);
2623           }
2624 
2625           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
2626           Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
2627           if (sidelineRegionDir != null) {
2628             sidelinedRegions.put(sidelineRegionDir, regionToSideline);
2629             LOG.info("After sidelined big overlapped region: "
2630               + regionToSideline.getRegionNameAsString()
2631               + " to " + sidelineRegionDir.toString());
2632             fixes++;
2633           }
2634         }
2635       }
2636     }
2637 
2638     /**
2639      * Check the region chain (from META) of this table.  We are looking for
2640      * holes, overlaps, and cycles.
2641      * @return false if there are errors
2642      * @throws IOException
2643      */
2644     public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
2645       // When table is disabled no need to check for the region chain. Some of the regions
2646       // accidently if deployed, this below code might report some issues like missing start
2647       // or end regions or region hole in chain and may try to fix which is unwanted.
2648       if (disabledTables.contains(this.tableName)) {
2649         return true;
2650       }
2651       int originalErrorsCount = errors.getErrorList().size();
2652       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
2653       SortedSet<byte[]> splits = sc.getSplits();
2654 
2655       byte[] prevKey = null;
2656       byte[] problemKey = null;
2657 
2658       if (splits.size() == 0) {
2659         // no region for this table
2660         handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
2661       }
2662 
2663       for (byte[] key : splits) {
2664         Collection<HbckInfo> ranges = regions.get(key);
2665         if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
2666           for (HbckInfo rng : ranges) {
2667             handler.handleRegionStartKeyNotEmpty(rng);
2668           }
2669         }
2670 
2671         // check for degenerate ranges
2672         for (HbckInfo rng : ranges) {
2673           // special endkey case converts '' to null
2674           byte[] endKey = rng.getEndKey();
2675           endKey = (endKey.length == 0) ? null : endKey;
2676           if (Bytes.equals(rng.getStartKey(),endKey)) {
2677             handler.handleDegenerateRegion(rng);
2678           }
2679         }
2680 
2681         if (ranges.size() == 1) {
2682           // this split key is ok -- no overlap, not a hole.
2683           if (problemKey != null) {
2684             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2685           }
2686           problemKey = null; // fell through, no more problem.
2687         } else if (ranges.size() > 1) {
2688           // set the new problem key group name, if already have problem key, just
2689           // keep using it.
2690           if (problemKey == null) {
2691             // only for overlap regions.
2692             LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
2693             problemKey = key;
2694           }
2695           overlapGroups.putAll(problemKey, ranges);
2696 
2697           // record errors
2698           ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
2699           //  this dumb and n^2 but this shouldn't happen often
2700           for (HbckInfo r1 : ranges) {
2701             if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2702             subRange.remove(r1);
2703             for (HbckInfo r2 : subRange) {
2704               if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2705               if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
2706                 handler.handleDuplicateStartKeys(r1,r2);
2707               } else {
2708                 // overlap
2709                 handler.handleOverlapInRegionChain(r1, r2);
2710               }
2711             }
2712           }
2713 
2714         } else if (ranges.size() == 0) {
2715           if (problemKey != null) {
2716             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2717           }
2718           problemKey = null;
2719 
2720           byte[] holeStopKey = sc.getSplits().higher(key);
2721           // if higher key is null we reached the top.
2722           if (holeStopKey != null) {
2723             // hole
2724             handler.handleHoleInRegionChain(key, holeStopKey);
2725           }
2726         }
2727         prevKey = key;
2728       }
2729 
2730       // When the last region of a table is proper and having an empty end key, 'prevKey'
2731       // will be null.
2732       if (prevKey != null) {
2733         handler.handleRegionEndKeyNotEmpty(prevKey);
2734       }
2735 
2736       // TODO fold this into the TableIntegrityHandler
2737       if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
2738         LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
2739             " false to run serially.");
2740         boolean ok = handleOverlapsParallel(handler, prevKey);
2741         if (!ok) {
2742           return false;
2743         }
2744       } else {
2745         LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
2746             " true to run in parallel.");
2747         for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2748           handler.handleOverlapGroup(overlap);
2749         }
2750       }
2751 
2752       if (details) {
2753         // do full region split map dump
2754         errors.print("---- Table '"  +  this.tableName
2755             + "': region split map");
2756         dump(splits, regions);
2757         errors.print("---- Table '"  +  this.tableName
2758             + "': overlap groups");
2759         dumpOverlapProblems(overlapGroups);
2760         errors.print("There are " + overlapGroups.keySet().size()
2761             + " overlap groups with " + overlapGroups.size()
2762             + " overlapping regions");
2763       }
2764       if (!sidelinedRegions.isEmpty()) {
2765         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
2766         errors.print("---- Table '"  +  this.tableName
2767             + "': sidelined big overlapped regions");
2768         dumpSidelinedRegions(sidelinedRegions);
2769       }
2770       return errors.getErrorList().size() == originalErrorsCount;
2771     }
2772 
2773     private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
2774         throws IOException {
2775       // we parallelize overlap handler for the case we have lots of groups to fix.  We can
2776       // safely assume each group is independent.
2777       List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
2778       List<Future<Void>> rets;
2779       for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2780         //
2781         merges.add(new WorkItemOverlapMerge(overlap, handler));
2782       }
2783       try {
2784         rets = executor.invokeAll(merges);
2785       } catch (InterruptedException e) {
2786         LOG.error("Overlap merges were interrupted", e);
2787         return false;
2788       }
2789       for(int i=0; i<merges.size(); i++) {
2790         WorkItemOverlapMerge work = merges.get(i);
2791         Future<Void> f = rets.get(i);
2792         try {
2793           f.get();
2794         } catch(ExecutionException e) {
2795           LOG.warn("Failed to merge overlap group" + work, e.getCause());
2796         } catch (InterruptedException e) {
2797           LOG.error("Waiting for overlap merges was interrupted", e);
2798           return false;
2799         }
2800       }
2801       return true;
2802     }
2803 
2804     /**
2805      * This dumps data in a visually reasonable way for visual debugging
2806      *
2807      * @param splits
2808      * @param regions
2809      */
2810     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
2811       // we display this way because the last end key should be displayed as well.
2812       StringBuilder sb = new StringBuilder();
2813       for (byte[] k : splits) {
2814         sb.setLength(0); // clear out existing buffer, if any.
2815         sb.append(Bytes.toStringBinary(k) + ":\t");
2816         for (HbckInfo r : regions.get(k)) {
2817           sb.append("[ "+ r.toString() + ", "
2818               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
2819         }
2820         errors.print(sb.toString());
2821       }
2822     }
2823   }
2824 
2825   public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
2826     // we display this way because the last end key should be displayed as
2827     // well.
2828     for (byte[] k : regions.keySet()) {
2829       errors.print(Bytes.toStringBinary(k) + ":");
2830       for (HbckInfo r : regions.get(k)) {
2831         errors.print("[ " + r.toString() + ", "
2832             + Bytes.toStringBinary(r.getEndKey()) + "]");
2833       }
2834       errors.print("----");
2835     }
2836   }
2837 
2838   public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
2839     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
2840       TableName tableName = entry.getValue().getTableName();
2841       Path path = entry.getKey();
2842       errors.print("This sidelined region dir should be bulk loaded: "
2843         + path.toString());
2844       errors.print("Bulk load command looks like: "
2845         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
2846         + path.toUri().getPath() + " "+ tableName);
2847     }
2848   }
2849 
2850   public Multimap<byte[], HbckInfo> getOverlapGroups(
2851       TableName table) {
2852     TableInfo ti = tablesInfo.get(table);
2853     return ti.overlapGroups;
2854   }
2855 
2856   /**
2857    * Return a list of user-space table names whose metadata have not been
2858    * modified in the last few milliseconds specified by timelag
2859    * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
2860    * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
2861    * milliseconds specified by timelag, then the table is a candidate to be returned.
2862    * @return tables that have not been modified recently
2863    * @throws IOException if an error is encountered
2864    */
2865   HTableDescriptor[] getTables(AtomicInteger numSkipped) {
2866     List<TableName> tableNames = new ArrayList<TableName>();
2867     long now = System.currentTimeMillis();
2868 
2869     for (HbckInfo hbi : regionInfoMap.values()) {
2870       MetaEntry info = hbi.metaEntry;
2871 
2872       // if the start key is zero, then we have found the first region of a table.
2873       // pick only those tables that were not modified in the last few milliseconds.
2874       if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
2875         if (info.modTime + timelag < now) {
2876           tableNames.add(info.getTable());
2877         } else {
2878           numSkipped.incrementAndGet(); // one more in-flux table
2879         }
2880       }
2881     }
2882     return getHTableDescriptors(tableNames);
2883   }
2884 
2885   HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
2886     HTableDescriptor[] htd = new HTableDescriptor[0];
2887     Admin admin = null;
2888     try {
2889       LOG.info("getHTableDescriptors == tableNames => " + tableNames);
2890       admin = new HBaseAdmin(getConf());
2891       htd = admin.getTableDescriptorsByTableName(tableNames);
2892     } catch (IOException e) {
2893       LOG.debug("Exception getting table descriptors", e);
2894     } finally {
2895       if (admin != null) {
2896         try {
2897           admin.close();
2898         } catch (IOException e) {
2899           LOG.debug("Exception closing HBaseAdmin", e);
2900         }
2901       }
2902     }
2903     return htd;
2904   }
2905 
2906   /**
2907    * Gets the entry in regionInfo corresponding to the the given encoded
2908    * region name. If the region has not been seen yet, a new entry is added
2909    * and returned.
2910    */
2911   private synchronized HbckInfo getOrCreateInfo(String name) {
2912     HbckInfo hbi = regionInfoMap.get(name);
2913     if (hbi == null) {
2914       hbi = new HbckInfo(null);
2915       regionInfoMap.put(name, hbi);
2916     }
2917     return hbi;
2918   }
2919 
2920   private void checkAndFixTableLocks() throws IOException {
2921     TableLockChecker checker = new TableLockChecker(createZooKeeperWatcher(), errors);
2922     checker.checkTableLocks();
2923 
2924     if (this.fixTableLocks) {
2925       checker.fixExpiredTableLocks();
2926     }
2927   }
2928 
2929   /**
2930     * Check values in regionInfo for hbase:meta
2931     * Check if zero or more than one regions with hbase:meta are found.
2932     * If there are inconsistencies (i.e. zero or more than one regions
2933     * pretend to be holding the hbase:meta) try to fix that and report an error.
2934     * @throws IOException from HBaseFsckRepair functions
2935    * @throws KeeperException
2936    * @throws InterruptedException
2937     */
2938   boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
2939     List<HbckInfo> metaRegions = Lists.newArrayList();
2940     for (HbckInfo value : regionInfoMap.values()) {
2941       if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
2942         metaRegions.add(value);
2943       }
2944     }
2945 
2946     // There will be always one entry in regionInfoMap corresponding to hbase:meta
2947     // Check the deployed servers. It should be exactly one server.
2948     List<ServerName> servers = new ArrayList<ServerName>();
2949     HbckInfo metaHbckInfo = null;
2950     if (!metaRegions.isEmpty()) {
2951       metaHbckInfo = metaRegions.get(0);
2952       servers = metaHbckInfo.deployedOn;
2953     }
2954     if (servers.size() != 1) {
2955       if (servers.size() == 0) {
2956         errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta is not found on any region.");
2957         if (shouldFixAssignments()) {
2958           errors.print("Trying to fix a problem with hbase:meta..");
2959           setShouldRerun();
2960           // try to fix it (treat it as unassigned region)
2961           HBaseFsckRepair.fixUnassigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
2962           HBaseFsckRepair.waitUntilAssigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
2963         }
2964       } else if (servers.size() > 1) {
2965         errors
2966             .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta is found on more than one region.");
2967         if (shouldFixAssignments()) {
2968           if (metaHbckInfo == null) {
2969             errors.print(
2970               "Unable to fix problem with hbase:meta due to hbase:meta region info missing");
2971             return false;
2972           }
2973           errors.print("Trying to fix a problem with hbase:meta..");
2974           setShouldRerun();
2975           // try fix it (treat is a dupe assignment)
2976           HBaseFsckRepair.fixMultiAssignment(admin, metaHbckInfo.metaEntry, servers);
2977         }
2978       }
2979       // rerun hbck with hopefully fixed META
2980       return false;
2981     }
2982     // no errors, so continue normally
2983     return true;
2984   }
2985 
2986   /**
2987    * Scan hbase:meta, adding all regions found to the regionInfo map.
2988    * @throws IOException if an error is encountered
2989    */
2990   boolean loadMetaEntries() throws IOException {
2991     MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
2992       int countRecord = 1;
2993 
2994       // comparator to sort KeyValues with latest modtime
2995       final Comparator<Cell> comp = new Comparator<Cell>() {
2996         @Override
2997         public int compare(Cell k1, Cell k2) {
2998           return (int)(k1.getTimestamp() - k2.getTimestamp());
2999         }
3000       };
3001 
3002       @Override
3003       public boolean processRow(Result result) throws IOException {
3004         try {
3005 
3006           // record the latest modification of this META record
3007           long ts =  Collections.max(result.listCells(), comp).getTimestamp();
3008           RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
3009           if (rl == null) {
3010             emptyRegionInfoQualifiers.add(result);
3011             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3012               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3013             return true;
3014           }
3015           ServerName sn = null;
3016           if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
3017               rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
3018             emptyRegionInfoQualifiers.add(result);
3019             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3020               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3021             return true;
3022           }
3023           HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
3024           if (!(isTableIncluded(hri.getTable())
3025               || hri.isMetaRegion())) {
3026             return true;
3027           }
3028           PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
3029           for (HRegionLocation h : rl.getRegionLocations()) {
3030             if (h == null || h.getRegionInfo() == null) {
3031               continue;
3032             }
3033             sn = h.getServerName();
3034             hri = h.getRegionInfo();
3035 
3036             MetaEntry m = null;
3037             if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
3038               m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
3039             } else {
3040               m = new MetaEntry(hri, sn, ts, null, null);
3041             }
3042             HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
3043             if (previous == null) {
3044               regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
3045             } else if (previous.metaEntry == null) {
3046               previous.metaEntry = m;
3047             } else {
3048               throw new IOException("Two entries in hbase:meta are same " + previous);
3049             }
3050           }
3051           PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
3052           for (HRegionInfo mergeRegion : new HRegionInfo[] {
3053               mergeRegions.getFirst(), mergeRegions.getSecond() }) {
3054             if (mergeRegion != null) {
3055               // This region is already been merged
3056               HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
3057               hbInfo.setMerged(true);
3058             }
3059           }
3060 
3061           // show proof of progress to the user, once for every 100 records.
3062           if (countRecord % 100 == 0) {
3063             errors.progress();
3064           }
3065           countRecord++;
3066           return true;
3067         } catch (RuntimeException e) {
3068           LOG.error("Result=" + result);
3069           throw e;
3070         }
3071       }
3072     };
3073     if (!checkMetaOnly) {
3074       // Scan hbase:meta to pick up user regions
3075       MetaScanner.metaScan(getConf(), visitor);
3076     }
3077 
3078     errors.print("");
3079     return true;
3080   }
3081 
3082   /**
3083    * Stores the regioninfo entries scanned from META
3084    */
3085   static class MetaEntry extends HRegionInfo {
3086     ServerName regionServer;   // server hosting this region
3087     long modTime;          // timestamp of most recent modification metadata
3088     HRegionInfo splitA, splitB; //split daughters
3089 
3090     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3091       this(rinfo, regionServer, modTime, null, null);
3092     }
3093 
3094     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3095         HRegionInfo splitA, HRegionInfo splitB) {
3096       super(rinfo);
3097       this.regionServer = regionServer;
3098       this.modTime = modTime;
3099       this.splitA = splitA;
3100       this.splitB = splitB;
3101     }
3102 
3103     @Override
3104     public boolean equals(Object o) {
3105       boolean superEq = super.equals(o);
3106       if (!superEq) {
3107         return superEq;
3108       }
3109 
3110       MetaEntry me = (MetaEntry) o;
3111       if (!regionServer.equals(me.regionServer)) {
3112         return false;
3113       }
3114       return (modTime == me.modTime);
3115     }
3116 
3117     @Override
3118     public int hashCode() {
3119       int hash = Arrays.hashCode(getRegionName());
3120       hash ^= getRegionId();
3121       hash ^= Arrays.hashCode(getStartKey());
3122       hash ^= Arrays.hashCode(getEndKey());
3123       hash ^= Boolean.valueOf(isOffline()).hashCode();
3124       hash ^= getTable().hashCode();
3125       if (regionServer != null) {
3126         hash ^= regionServer.hashCode();
3127       }
3128       hash ^= modTime;
3129       return hash;
3130     }
3131   }
3132 
3133   /**
3134    * Stores the regioninfo entries from HDFS
3135    */
3136   static class HdfsEntry {
3137     HRegionInfo hri;
3138     Path hdfsRegionDir = null;
3139     long hdfsRegionDirModTime  = 0;
3140     boolean hdfsRegioninfoFilePresent = false;
3141     boolean hdfsOnlyEdits = false;
3142   }
3143 
3144   /**
3145    * Stores the regioninfo retrieved from Online region servers.
3146    */
3147   static class OnlineEntry {
3148     HRegionInfo hri;
3149     ServerName hsa;
3150 
3151     @Override
3152     public String toString() {
3153       return hsa.toString() + ";" + hri.getRegionNameAsString();
3154     }
3155   }
3156 
3157   /**
3158    * Maintain information about a particular region.  It gathers information
3159    * from three places -- HDFS, META, and region servers.
3160    */
3161   public static class HbckInfo implements KeyRange {
3162     private MetaEntry metaEntry = null; // info in META
3163     private HdfsEntry hdfsEntry = null; // info in HDFS
3164     private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
3165     private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
3166     private boolean skipChecks = false; // whether to skip further checks to this region info.
3167     private boolean isMerged = false;// whether this region has already been merged into another one
3168     private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3169     private HRegionInfo primaryHRIForDeployedReplica = null;
3170 
3171     HbckInfo(MetaEntry metaEntry) {
3172       this.metaEntry = metaEntry;
3173     }
3174 
3175     public int getReplicaId() {
3176       if (metaEntry != null) return metaEntry.getReplicaId();
3177       return deployedReplicaId;
3178     }
3179 
3180     public synchronized void addServer(HRegionInfo hri, ServerName server) {
3181       OnlineEntry rse = new OnlineEntry() ;
3182       rse.hri = hri;
3183       rse.hsa = server;
3184       this.deployedEntries.add(rse);
3185       this.deployedOn.add(server);
3186       // save the replicaId that we see deployed in the cluster
3187       this.deployedReplicaId = hri.getReplicaId();
3188       this.primaryHRIForDeployedReplica =
3189           RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3190     }
3191 
3192     @Override
3193     public synchronized String toString() {
3194       StringBuilder sb = new StringBuilder();
3195       sb.append("{ meta => ");
3196       sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3197       sb.append( ", hdfs => " + getHdfsRegionDir());
3198       sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3199       sb.append( ", replicaId => " + getReplicaId());
3200       sb.append(" }");
3201       return sb.toString();
3202     }
3203 
3204     @Override
3205     public byte[] getStartKey() {
3206       if (this.metaEntry != null) {
3207         return this.metaEntry.getStartKey();
3208       } else if (this.hdfsEntry != null) {
3209         return this.hdfsEntry.hri.getStartKey();
3210       } else {
3211         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3212         return null;
3213       }
3214     }
3215 
3216     @Override
3217     public byte[] getEndKey() {
3218       if (this.metaEntry != null) {
3219         return this.metaEntry.getEndKey();
3220       } else if (this.hdfsEntry != null) {
3221         return this.hdfsEntry.hri.getEndKey();
3222       } else {
3223         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3224         return null;
3225       }
3226     }
3227 
3228     public TableName getTableName() {
3229       if (this.metaEntry != null) {
3230         return this.metaEntry.getTable();
3231       } else if (this.hdfsEntry != null) {
3232         // we are only guaranteed to have a path and not an HRI for hdfsEntry,
3233         // so we get the name from the Path
3234         Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3235         return FSUtils.getTableName(tableDir);
3236       } else {
3237         // return the info from the first online/deployed hri
3238         for (OnlineEntry e : deployedEntries) {
3239           return e.hri.getTable();
3240         }
3241         return null;
3242       }
3243     }
3244 
3245     public String getRegionNameAsString() {
3246       if (metaEntry != null) {
3247         return metaEntry.getRegionNameAsString();
3248       } else if (hdfsEntry != null) {
3249         if (hdfsEntry.hri != null) {
3250           return hdfsEntry.hri.getRegionNameAsString();
3251         }
3252       } else {
3253         // return the info from the first online/deployed hri
3254         for (OnlineEntry e : deployedEntries) {
3255           return e.hri.getRegionNameAsString();
3256         }
3257       }
3258       return null;
3259     }
3260 
3261     public byte[] getRegionName() {
3262       if (metaEntry != null) {
3263         return metaEntry.getRegionName();
3264       } else if (hdfsEntry != null) {
3265         return hdfsEntry.hri.getRegionName();
3266       } else {
3267         // return the info from the first online/deployed hri
3268         for (OnlineEntry e : deployedEntries) {
3269           return e.hri.getRegionName();
3270         }
3271         return null;
3272       }
3273     }
3274 
3275     public HRegionInfo getPrimaryHRIForDeployedReplica() {
3276       return primaryHRIForDeployedReplica;
3277     }
3278 
3279     Path getHdfsRegionDir() {
3280       if (hdfsEntry == null) {
3281         return null;
3282       }
3283       return hdfsEntry.hdfsRegionDir;
3284     }
3285 
3286     boolean containsOnlyHdfsEdits() {
3287       if (hdfsEntry == null) {
3288         return false;
3289       }
3290       return hdfsEntry.hdfsOnlyEdits;
3291     }
3292 
3293     boolean isHdfsRegioninfoPresent() {
3294       if (hdfsEntry == null) {
3295         return false;
3296       }
3297       return hdfsEntry.hdfsRegioninfoFilePresent;
3298     }
3299 
3300     long getModTime() {
3301       if (hdfsEntry == null) {
3302         return 0;
3303       }
3304       return hdfsEntry.hdfsRegionDirModTime;
3305     }
3306 
3307     HRegionInfo getHdfsHRI() {
3308       if (hdfsEntry == null) {
3309         return null;
3310       }
3311       return hdfsEntry.hri;
3312     }
3313 
3314     public void setSkipChecks(boolean skipChecks) {
3315       this.skipChecks = skipChecks;
3316     }
3317 
3318     public boolean isSkipChecks() {
3319       return skipChecks;
3320     }
3321 
3322     public void setMerged(boolean isMerged) {
3323       this.isMerged = isMerged;
3324     }
3325 
3326     public boolean isMerged() {
3327       return this.isMerged;
3328     }
3329   }
3330 
3331   final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3332     @Override
3333     public int compare(HbckInfo l, HbckInfo r) {
3334       if (l == r) {
3335         // same instance
3336         return 0;
3337       }
3338 
3339       int tableCompare = l.getTableName().compareTo(r.getTableName());
3340       if (tableCompare != 0) {
3341         return tableCompare;
3342       }
3343 
3344       int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3345           l.getStartKey(), r.getStartKey());
3346       if (startComparison != 0) {
3347         return startComparison;
3348       }
3349 
3350       // Special case for absolute endkey
3351       byte[] endKey = r.getEndKey();
3352       endKey = (endKey.length == 0) ? null : endKey;
3353       byte[] endKey2 = l.getEndKey();
3354       endKey2 = (endKey2.length == 0) ? null : endKey2;
3355       int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3356           endKey2,  endKey);
3357 
3358       if (endComparison != 0) {
3359         return endComparison;
3360       }
3361 
3362       // use regionId as tiebreaker.
3363       // Null is considered after all possible values so make it bigger.
3364       if (l.hdfsEntry == null && r.hdfsEntry == null) {
3365         return 0;
3366       }
3367       if (l.hdfsEntry == null && r.hdfsEntry != null) {
3368         return 1;
3369       }
3370       // l.hdfsEntry must not be null
3371       if (r.hdfsEntry == null) {
3372         return -1;
3373       }
3374       // both l.hdfsEntry and r.hdfsEntry must not be null.
3375       return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3376     }
3377   };
3378 
3379   /**
3380    * Prints summary of all tables found on the system.
3381    */
3382   private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3383     StringBuilder sb = new StringBuilder();
3384     errors.print("Summary:");
3385     for (TableInfo tInfo : tablesInfo.values()) {
3386       if (errors.tableHasErrors(tInfo)) {
3387         errors.print("Table " + tInfo.getName() + " is inconsistent.");
3388       } else {
3389         errors.print("  " + tInfo.getName() + " is okay.");
3390       }
3391       errors.print("    Number of regions: " + tInfo.getNumRegions());
3392       sb.setLength(0); // clear out existing buffer, if any.
3393       sb.append("    Deployed on: ");
3394       for (ServerName server : tInfo.deployedOn) {
3395         sb.append(" " + server.toString());
3396       }
3397       errors.print(sb.toString());
3398     }
3399   }
3400 
3401   static ErrorReporter getErrorReporter(
3402       final Configuration conf) throws ClassNotFoundException {
3403     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3404     return ReflectionUtils.newInstance(reporter, conf);
3405   }
3406 
3407   public interface ErrorReporter {
3408     enum ERROR_CODE {
3409       UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3410       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
3411       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3412       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3413       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3414       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3415       WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR
3416     }
3417     void clear();
3418     void report(String message);
3419     void reportError(String message);
3420     void reportError(ERROR_CODE errorCode, String message);
3421     void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3422     void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3423     void reportError(
3424       ERROR_CODE errorCode,
3425       String message,
3426       TableInfo table,
3427       HbckInfo info1,
3428       HbckInfo info2
3429     );
3430     int summarize();
3431     void detail(String details);
3432     ArrayList<ERROR_CODE> getErrorList();
3433     void progress();
3434     void print(String message);
3435     void resetErrors();
3436     boolean tableHasErrors(TableInfo table);
3437   }
3438 
3439   static class PrintingErrorReporter implements ErrorReporter {
3440     public int errorCount = 0;
3441     private int showProgress;
3442 
3443     Set<TableInfo> errorTables = new HashSet<TableInfo>();
3444 
3445     // for use by unit tests to verify which errors were discovered
3446     private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3447 
3448     @Override
3449     public void clear() {
3450       errorTables.clear();
3451       errorList.clear();
3452       errorCount = 0;
3453     }
3454 
3455     @Override
3456     public synchronized void reportError(ERROR_CODE errorCode, String message) {
3457       if (errorCode == ERROR_CODE.WRONG_USAGE) {
3458         System.err.println(message);
3459         return;
3460       }
3461 
3462       errorList.add(errorCode);
3463       if (!summary) {
3464         System.out.println("ERROR: " + message);
3465       }
3466       errorCount++;
3467       showProgress = 0;
3468     }
3469 
3470     @Override
3471     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
3472       errorTables.add(table);
3473       reportError(errorCode, message);
3474     }
3475 
3476     @Override
3477     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3478                                          HbckInfo info) {
3479       errorTables.add(table);
3480       String reference = "(region " + info.getRegionNameAsString() + ")";
3481       reportError(errorCode, reference + " " + message);
3482     }
3483 
3484     @Override
3485     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3486                                          HbckInfo info1, HbckInfo info2) {
3487       errorTables.add(table);
3488       String reference = "(regions " + info1.getRegionNameAsString()
3489           + " and " + info2.getRegionNameAsString() + ")";
3490       reportError(errorCode, reference + " " + message);
3491     }
3492 
3493     @Override
3494     public synchronized void reportError(String message) {
3495       reportError(ERROR_CODE.UNKNOWN, message);
3496     }
3497 
3498     /**
3499      * Report error information, but do not increment the error count.  Intended for cases
3500      * where the actual error would have been reported previously.
3501      * @param message
3502      */
3503     @Override
3504     public synchronized void report(String message) {
3505       if (! summary) {
3506         System.out.println("ERROR: " + message);
3507       }
3508       showProgress = 0;
3509     }
3510 
3511     @Override
3512     public synchronized int summarize() {
3513       System.out.println(Integer.toString(errorCount) +
3514                          " inconsistencies detected.");
3515       if (errorCount == 0) {
3516         System.out.println("Status: OK");
3517         return 0;
3518       } else {
3519         System.out.println("Status: INCONSISTENT");
3520         return -1;
3521       }
3522     }
3523 
3524     @Override
3525     public ArrayList<ERROR_CODE> getErrorList() {
3526       return errorList;
3527     }
3528 
3529     @Override
3530     public synchronized void print(String message) {
3531       if (!summary) {
3532         System.out.println(message);
3533       }
3534     }
3535 
3536     @Override
3537     public boolean tableHasErrors(TableInfo table) {
3538       return errorTables.contains(table);
3539     }
3540 
3541     @Override
3542     public void resetErrors() {
3543       errorCount = 0;
3544     }
3545 
3546     @Override
3547     public synchronized void detail(String message) {
3548       if (details) {
3549         System.out.println(message);
3550       }
3551       showProgress = 0;
3552     }
3553 
3554     @Override
3555     public synchronized void progress() {
3556       if (showProgress++ == 10) {
3557         if (!summary) {
3558           System.out.print(".");
3559         }
3560         showProgress = 0;
3561       }
3562     }
3563   }
3564 
3565   /**
3566    * Contact a region server and get all information from it
3567    */
3568   static class WorkItemRegion implements Callable<Void> {
3569     private HBaseFsck hbck;
3570     private ServerName rsinfo;
3571     private ErrorReporter errors;
3572     private HConnection connection;
3573 
3574     WorkItemRegion(HBaseFsck hbck, ServerName info,
3575                    ErrorReporter errors, HConnection connection) {
3576       this.hbck = hbck;
3577       this.rsinfo = info;
3578       this.errors = errors;
3579       this.connection = connection;
3580     }
3581 
3582     @Override
3583     public synchronized Void call() throws IOException {
3584       errors.progress();
3585       try {
3586         BlockingInterface server = connection.getAdmin(rsinfo);
3587 
3588         // list all online regions from this region server
3589         List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
3590         regions = filterRegions(regions);
3591 
3592         if (details) {
3593           errors.detail("RegionServer: " + rsinfo.getServerName() +
3594                            " number of regions: " + regions.size());
3595           for (HRegionInfo rinfo: regions) {
3596             errors.detail("  " + rinfo.getRegionNameAsString() +
3597                              " id: " + rinfo.getRegionId() +
3598                              " encoded_name: " + rinfo.getEncodedName() +
3599                              " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
3600                              " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
3601           }
3602         }
3603 
3604         // check to see if the existence of this region matches the region in META
3605         for (HRegionInfo r:regions) {
3606           HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
3607           hbi.addServer(r, rsinfo);
3608         }
3609       } catch (IOException e) {          // unable to connect to the region server.
3610         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
3611           " Unable to fetch region information. " + e);
3612         throw e;
3613       }
3614       return null;
3615     }
3616 
3617     private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
3618       List<HRegionInfo> ret = Lists.newArrayList();
3619       for (HRegionInfo hri : regions) {
3620         if (hri.isMetaTable() || (!hbck.checkMetaOnly
3621             && hbck.isTableIncluded(hri.getTable()))) {
3622           ret.add(hri);
3623         }
3624       }
3625       return ret;
3626     }
3627   }
3628 
3629   /**
3630    * Contact hdfs and get all information about specified table directory into
3631    * regioninfo list.
3632    */
3633   static class WorkItemHdfsDir implements Callable<Void> {
3634     private HBaseFsck hbck;
3635     private FileStatus tableDir;
3636     private ErrorReporter errors;
3637     private FileSystem fs;
3638 
3639     WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
3640                     FileStatus status) {
3641       this.hbck = hbck;
3642       this.fs = fs;
3643       this.tableDir = status;
3644       this.errors = errors;
3645     }
3646 
3647     @Override
3648     public synchronized Void call() throws IOException {
3649       try {
3650         // level 2: <HBASE_DIR>/<table>/*
3651         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
3652         for (FileStatus regionDir : regionDirs) {
3653           String encodedName = regionDir.getPath().getName();
3654           // ignore directories that aren't hexadecimal
3655           if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
3656             continue;
3657           }
3658 
3659           LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
3660           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
3661           HdfsEntry he = new HdfsEntry();
3662           synchronized (hbi) {
3663             if (hbi.getHdfsRegionDir() != null) {
3664               errors.print("Directory " + encodedName + " duplicate??" +
3665                            hbi.getHdfsRegionDir());
3666             }
3667 
3668             he.hdfsRegionDir = regionDir.getPath();
3669             he.hdfsRegionDirModTime = regionDir.getModificationTime();
3670             Path regioninfoFile = new Path(he.hdfsRegionDir, HRegionFileSystem.REGION_INFO_FILE);
3671             he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
3672             // we add to orphan list when we attempt to read .regioninfo
3673 
3674             // Set a flag if this region contains only edits
3675             // This is special case if a region is left after split
3676             he.hdfsOnlyEdits = true;
3677             FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
3678             Path ePath = HLogUtil.getRegionDirRecoveredEditsDir(regionDir.getPath());
3679             for (FileStatus subDir : subDirs) {
3680               String sdName = subDir.getPath().getName();
3681               if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
3682                 he.hdfsOnlyEdits = false;
3683                 break;
3684               }
3685             }
3686             hbi.hdfsEntry = he;
3687           }
3688         }
3689       } catch (IOException e) {
3690         // unable to connect to the region server.
3691         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
3692             + tableDir.getPath().getName()
3693             + " Unable to fetch region information. " + e);
3694         throw e;
3695       }
3696       return null;
3697     }
3698   }
3699 
3700   /**
3701    * Contact hdfs and get all information about specified table directory into
3702    * regioninfo list.
3703    */
3704   static class WorkItemHdfsRegionInfo implements Callable<Void> {
3705     private HbckInfo hbi;
3706     private HBaseFsck hbck;
3707     private ErrorReporter errors;
3708 
3709     WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
3710       this.hbi = hbi;
3711       this.hbck = hbck;
3712       this.errors = errors;
3713     }
3714 
3715     @Override
3716     public synchronized Void call() throws IOException {
3717       // only load entries that haven't been loaded yet.
3718       if (hbi.getHdfsHRI() == null) {
3719         try {
3720           hbck.loadHdfsRegioninfo(hbi);
3721         } catch (IOException ioe) {
3722           String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
3723               + hbi.getTableName() + " in hdfs dir "
3724               + hbi.getHdfsRegionDir()
3725               + "!  It may be an invalid format or version file.  Treating as "
3726               + "an orphaned regiondir.";
3727           errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
3728           try {
3729             hbck.debugLsr(hbi.getHdfsRegionDir());
3730           } catch (IOException ioe2) {
3731             LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
3732             throw ioe2;
3733           }
3734           hbck.orphanHdfsDirs.add(hbi);
3735           throw ioe;
3736         }
3737       }
3738       return null;
3739     }
3740   };
3741 
3742   /**
3743    * Display the full report from fsck. This displays all live and dead region
3744    * servers, and all known regions.
3745    */
3746   public void setDisplayFullReport() {
3747     details = true;
3748   }
3749 
3750   /**
3751    * Set summary mode.
3752    * Print only summary of the tables and status (OK or INCONSISTENT)
3753    */
3754   void setSummary() {
3755     summary = true;
3756   }
3757 
3758   /**
3759    * Set hbase:meta check mode.
3760    * Print only info about hbase:meta table deployment/state
3761    */
3762   void setCheckMetaOnly() {
3763     checkMetaOnly = true;
3764   }
3765 
3766   /**
3767    * Set region boundaries check mode.
3768    */
3769   void setRegionBoundariesCheck() {
3770     checkRegionBoundaries = true;
3771   }
3772 
3773   /**
3774    * Set table locks fix mode.
3775    * Delete table locks held for a long time
3776    */
3777   public void setFixTableLocks(boolean shouldFix) {
3778     fixTableLocks = shouldFix;
3779     fixAny |= shouldFix;
3780   }
3781 
3782   /**
3783    * Check if we should rerun fsck again. This checks if we've tried to
3784    * fix something and we should rerun fsck tool again.
3785    * Display the full report from fsck. This displays all live and dead
3786    * region servers, and all known regions.
3787    */
3788   void setShouldRerun() {
3789     rerun = true;
3790   }
3791 
3792   boolean shouldRerun() {
3793     return rerun;
3794   }
3795 
3796   /**
3797    * Fix inconsistencies found by fsck. This should try to fix errors (if any)
3798    * found by fsck utility.
3799    */
3800   public void setFixAssignments(boolean shouldFix) {
3801     fixAssignments = shouldFix;
3802     fixAny |= shouldFix;
3803   }
3804 
3805   boolean shouldFixAssignments() {
3806     return fixAssignments;
3807   }
3808 
3809   public void setFixMeta(boolean shouldFix) {
3810     fixMeta = shouldFix;
3811     fixAny |= shouldFix;
3812   }
3813 
3814   boolean shouldFixMeta() {
3815     return fixMeta;
3816   }
3817 
3818   public void setFixEmptyMetaCells(boolean shouldFix) {
3819     fixEmptyMetaCells = shouldFix;
3820     fixAny |= shouldFix;
3821   }
3822 
3823   boolean shouldFixEmptyMetaCells() {
3824     return fixEmptyMetaCells;
3825   }
3826 
3827   public void setCheckHdfs(boolean checking) {
3828     checkHdfs = checking;
3829   }
3830 
3831   boolean shouldCheckHdfs() {
3832     return checkHdfs;
3833   }
3834 
3835   public void setFixHdfsHoles(boolean shouldFix) {
3836     fixHdfsHoles = shouldFix;
3837     fixAny |= shouldFix;
3838   }
3839 
3840   boolean shouldFixHdfsHoles() {
3841     return fixHdfsHoles;
3842   }
3843 
3844   public void setFixTableOrphans(boolean shouldFix) {
3845     fixTableOrphans = shouldFix;
3846     fixAny |= shouldFix;
3847   }
3848 
3849   boolean shouldFixTableOrphans() {
3850     return fixTableOrphans;
3851   }
3852 
3853   public void setFixHdfsOverlaps(boolean shouldFix) {
3854     fixHdfsOverlaps = shouldFix;
3855     fixAny |= shouldFix;
3856   }
3857 
3858   boolean shouldFixHdfsOverlaps() {
3859     return fixHdfsOverlaps;
3860   }
3861 
3862   public void setFixHdfsOrphans(boolean shouldFix) {
3863     fixHdfsOrphans = shouldFix;
3864     fixAny |= shouldFix;
3865   }
3866 
3867   boolean shouldFixHdfsOrphans() {
3868     return fixHdfsOrphans;
3869   }
3870 
3871   public void setFixVersionFile(boolean shouldFix) {
3872     fixVersionFile = shouldFix;
3873     fixAny |= shouldFix;
3874   }
3875 
3876   public boolean shouldFixVersionFile() {
3877     return fixVersionFile;
3878   }
3879 
3880   public void setSidelineBigOverlaps(boolean sbo) {
3881     this.sidelineBigOverlaps = sbo;
3882   }
3883 
3884   public boolean shouldSidelineBigOverlaps() {
3885     return sidelineBigOverlaps;
3886   }
3887 
3888   public void setFixSplitParents(boolean shouldFix) {
3889     fixSplitParents = shouldFix;
3890     fixAny |= shouldFix;
3891   }
3892 
3893   boolean shouldFixSplitParents() {
3894     return fixSplitParents;
3895   }
3896 
3897   public void setFixReferenceFiles(boolean shouldFix) {
3898     fixReferenceFiles = shouldFix;
3899     fixAny |= shouldFix;
3900   }
3901 
3902   boolean shouldFixReferenceFiles() {
3903     return fixReferenceFiles;
3904   }
3905 
3906   public boolean shouldIgnorePreCheckPermission() {
3907     return !fixAny || ignorePreCheckPermission;
3908   }
3909 
3910   public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
3911     this.ignorePreCheckPermission = ignorePreCheckPermission;
3912   }
3913 
3914   /**
3915    * @param mm maximum number of regions to merge into a single region.
3916    */
3917   public void setMaxMerge(int mm) {
3918     this.maxMerge = mm;
3919   }
3920 
3921   public int getMaxMerge() {
3922     return maxMerge;
3923   }
3924 
3925   public void setMaxOverlapsToSideline(int mo) {
3926     this.maxOverlapsToSideline = mo;
3927   }
3928 
3929   public int getMaxOverlapsToSideline() {
3930     return maxOverlapsToSideline;
3931   }
3932 
3933   /**
3934    * Only check/fix tables specified by the list,
3935    * Empty list means all tables are included.
3936    */
3937   boolean isTableIncluded(TableName table) {
3938     return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
3939   }
3940 
3941   public void includeTable(TableName table) {
3942     tablesIncluded.add(table);
3943   }
3944 
3945   Set<TableName> getIncludedTables() {
3946     return new HashSet<TableName>(tablesIncluded);
3947   }
3948 
3949   /**
3950    * We are interested in only those tables that have not changed their state in
3951    * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
3952    * @param seconds - the time in seconds
3953    */
3954   public void setTimeLag(long seconds) {
3955     timelag = seconds * 1000; // convert to milliseconds
3956   }
3957 
3958   /**
3959    *
3960    * @param sidelineDir - HDFS path to sideline data
3961    */
3962   public void setSidelineDir(String sidelineDir) {
3963     this.sidelineDir = new Path(sidelineDir);
3964   }
3965 
3966   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
3967     return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
3968   }
3969 
3970   public HFileCorruptionChecker getHFilecorruptionChecker() {
3971     return hfcc;
3972   }
3973 
3974   public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
3975     this.hfcc = hfcc;
3976   }
3977 
3978   public void setRetCode(int code) {
3979     this.retcode = code;
3980   }
3981 
3982   public int getRetCode() {
3983     return retcode;
3984   }
3985 
3986   protected HBaseFsck printUsageAndExit() {
3987     StringWriter sw = new StringWriter(2048);
3988     PrintWriter out = new PrintWriter(sw);
3989     out.println("Usage: fsck [opts] {only tables}");
3990     out.println(" where [opts] are:");
3991     out.println("   -help Display help options (this)");
3992     out.println("   -details Display full report of all regions.");
3993     out.println("   -timelag <timeInSeconds>  Process only regions that " +
3994                        " have not experienced any metadata updates in the last " +
3995                        " <timeInSeconds> seconds.");
3996     out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
3997         " before checking if the fix worked if run with -fix");
3998     out.println("   -summary Print only summary of the tables and status.");
3999     out.println("   -metaonly Only check the state of the hbase:meta table.");
4000     out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
4001     out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
4002 
4003     out.println("");
4004     out.println("  Metadata Repair options: (expert features, use with caution!)");
4005     out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
4006     out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
4007     out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
4008     out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
4009         + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
4010     out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
4011     out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
4012     out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
4013     out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
4014     out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
4015     out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
4016     out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
4017     out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
4018     out.println("   -fixSplitParents  Try to force offline split parents to be online.");
4019     out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
4020     out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
4021     out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
4022         + " (empty REGIONINFO_QUALIFIER rows)");
4023 
4024     out.println("");
4025     out.println("  Datafile Repair options: (expert features, use with caution!)");
4026     out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
4027     out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
4028 
4029     out.println("");
4030     out.println("  Metadata Repair shortcuts");
4031     out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
4032         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles -fixTableLocks");
4033     out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
4034 
4035     out.println("");
4036     out.println("  Table lock options");
4037     out.println("   -fixTableLocks    Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
4038 
4039     out.flush();
4040     errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
4041 
4042     setRetCode(-2);
4043     return this;
4044   }
4045 
4046   /**
4047    * Main program
4048    *
4049    * @param args
4050    * @throws Exception
4051    */
4052   public static void main(String[] args) throws Exception {
4053     // create a fsck object
4054     Configuration conf = HBaseConfiguration.create();
4055     Path hbasedir = FSUtils.getRootDir(conf);
4056     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
4057     FSUtils.setFsDefault(conf, new Path(defaultFs));
4058     int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
4059     System.exit(ret);
4060   }
4061 
4062   /**
4063    * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
4064    */
4065   static class HBaseFsckTool extends Configured implements Tool {
4066     HBaseFsckTool(Configuration conf) { super(conf); }
4067     @Override
4068     public int run(String[] args) throws Exception {
4069       HBaseFsck hbck = new HBaseFsck(getConf());
4070       hbck.exec(hbck.executor, args);
4071       return hbck.getRetCode();
4072     }
4073   };
4074 
4075 
4076   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
4077     ServiceException, InterruptedException {
4078     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
4079 
4080     boolean checkCorruptHFiles = false;
4081     boolean sidelineCorruptHFiles = false;
4082 
4083     // Process command-line args.
4084     for (int i = 0; i < args.length; i++) {
4085       String cmd = args[i];
4086       if (cmd.equals("-help") || cmd.equals("-h")) {
4087         return printUsageAndExit();
4088       } else if (cmd.equals("-details")) {
4089         setDisplayFullReport();
4090       } else if (cmd.equals("-timelag")) {
4091         if (i == args.length - 1) {
4092           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4093           return printUsageAndExit();
4094         }
4095         try {
4096           long timelag = Long.parseLong(args[i+1]);
4097           setTimeLag(timelag);
4098         } catch (NumberFormatException e) {
4099           errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4100           return printUsageAndExit();
4101         }
4102         i++;
4103       } else if (cmd.equals("-sleepBeforeRerun")) {
4104         if (i == args.length - 1) {
4105           errors.reportError(ERROR_CODE.WRONG_USAGE,
4106             "HBaseFsck: -sleepBeforeRerun needs a value.");
4107           return printUsageAndExit();
4108         }
4109         try {
4110           sleepBeforeRerun = Long.parseLong(args[i+1]);
4111         } catch (NumberFormatException e) {
4112           errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4113           return printUsageAndExit();
4114         }
4115         i++;
4116       } else if (cmd.equals("-sidelineDir")) {
4117         if (i == args.length - 1) {
4118           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4119           return printUsageAndExit();
4120         }
4121         i++;
4122         setSidelineDir(args[i]);
4123       } else if (cmd.equals("-fix")) {
4124         errors.reportError(ERROR_CODE.WRONG_USAGE,
4125           "This option is deprecated, please use  -fixAssignments instead.");
4126         setFixAssignments(true);
4127       } else if (cmd.equals("-fixAssignments")) {
4128         setFixAssignments(true);
4129       } else if (cmd.equals("-fixMeta")) {
4130         setFixMeta(true);
4131       } else if (cmd.equals("-noHdfsChecking")) {
4132         setCheckHdfs(false);
4133       } else if (cmd.equals("-fixHdfsHoles")) {
4134         setFixHdfsHoles(true);
4135       } else if (cmd.equals("-fixHdfsOrphans")) {
4136         setFixHdfsOrphans(true);
4137       } else if (cmd.equals("-fixTableOrphans")) {
4138         setFixTableOrphans(true);
4139       } else if (cmd.equals("-fixHdfsOverlaps")) {
4140         setFixHdfsOverlaps(true);
4141       } else if (cmd.equals("-fixVersionFile")) {
4142         setFixVersionFile(true);
4143       } else if (cmd.equals("-sidelineBigOverlaps")) {
4144         setSidelineBigOverlaps(true);
4145       } else if (cmd.equals("-fixSplitParents")) {
4146         setFixSplitParents(true);
4147       } else if (cmd.equals("-ignorePreCheckPermission")) {
4148         setIgnorePreCheckPermission(true);
4149       } else if (cmd.equals("-checkCorruptHFiles")) {
4150         checkCorruptHFiles = true;
4151       } else if (cmd.equals("-sidelineCorruptHFiles")) {
4152         sidelineCorruptHFiles = true;
4153       } else if (cmd.equals("-fixReferenceFiles")) {
4154         setFixReferenceFiles(true);
4155       } else if (cmd.equals("-fixEmptyMetaCells")) {
4156         setFixEmptyMetaCells(true);
4157       } else if (cmd.equals("-repair")) {
4158         // this attempts to merge overlapping hdfs regions, needs testing
4159         // under load
4160         setFixHdfsHoles(true);
4161         setFixHdfsOrphans(true);
4162         setFixMeta(true);
4163         setFixAssignments(true);
4164         setFixHdfsOverlaps(true);
4165         setFixVersionFile(true);
4166         setSidelineBigOverlaps(true);
4167         setFixSplitParents(false);
4168         setCheckHdfs(true);
4169         setFixReferenceFiles(true);
4170         setFixTableLocks(true);
4171       } else if (cmd.equals("-repairHoles")) {
4172         // this will make all missing hdfs regions available but may lose data
4173         setFixHdfsHoles(true);
4174         setFixHdfsOrphans(false);
4175         setFixMeta(true);
4176         setFixAssignments(true);
4177         setFixHdfsOverlaps(false);
4178         setSidelineBigOverlaps(false);
4179         setFixSplitParents(false);
4180         setCheckHdfs(true);
4181       } else if (cmd.equals("-maxOverlapsToSideline")) {
4182         if (i == args.length - 1) {
4183           errors.reportError(ERROR_CODE.WRONG_USAGE,
4184             "-maxOverlapsToSideline needs a numeric value argument.");
4185           return printUsageAndExit();
4186         }
4187         try {
4188           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4189           setMaxOverlapsToSideline(maxOverlapsToSideline);
4190         } catch (NumberFormatException e) {
4191           errors.reportError(ERROR_CODE.WRONG_USAGE,
4192             "-maxOverlapsToSideline needs a numeric value argument.");
4193           return printUsageAndExit();
4194         }
4195         i++;
4196       } else if (cmd.equals("-maxMerge")) {
4197         if (i == args.length - 1) {
4198           errors.reportError(ERROR_CODE.WRONG_USAGE,
4199             "-maxMerge needs a numeric value argument.");
4200           return printUsageAndExit();
4201         }
4202         try {
4203           int maxMerge = Integer.parseInt(args[i+1]);
4204           setMaxMerge(maxMerge);
4205         } catch (NumberFormatException e) {
4206           errors.reportError(ERROR_CODE.WRONG_USAGE,
4207             "-maxMerge needs a numeric value argument.");
4208           return printUsageAndExit();
4209         }
4210         i++;
4211       } else if (cmd.equals("-summary")) {
4212         setSummary();
4213       } else if (cmd.equals("-metaonly")) {
4214         setCheckMetaOnly();
4215       } else if (cmd.equals("-boundaries")) {
4216         setRegionBoundariesCheck();
4217       } else if (cmd.equals("-fixTableLocks")) {
4218         setFixTableLocks(true);
4219       } else if (cmd.startsWith("-")) {
4220         errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4221         return printUsageAndExit();
4222       } else {
4223         includeTable(TableName.valueOf(cmd));
4224         errors.print("Allow checking/fixes for table: " + cmd);
4225       }
4226     }
4227 
4228     // pre-check current user has FS write permission or not
4229     try {
4230       preCheckPermission();
4231     } catch (AccessDeniedException ace) {
4232       Runtime.getRuntime().exit(-1);
4233     } catch (IOException ioe) {
4234       Runtime.getRuntime().exit(-1);
4235     }
4236 
4237     // do the real work of hbck
4238     connect();
4239 
4240     try {
4241       // if corrupt file mode is on, first fix them since they may be opened later
4242       if (checkCorruptHFiles || sidelineCorruptHFiles) {
4243         LOG.info("Checking all hfiles for corruption");
4244         HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4245         setHFileCorruptionChecker(hfcc); // so we can get result
4246         Collection<TableName> tables = getIncludedTables();
4247         Collection<Path> tableDirs = new ArrayList<Path>();
4248         Path rootdir = FSUtils.getRootDir(getConf());
4249         if (tables.size() > 0) {
4250           for (TableName t : tables) {
4251             tableDirs.add(FSUtils.getTableDir(rootdir, t));
4252           }
4253         } else {
4254           tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4255         }
4256         hfcc.checkTables(tableDirs);
4257         hfcc.report(errors);
4258       }
4259 
4260       // check and fix table integrity, region consistency.
4261       int code = onlineHbck();
4262       setRetCode(code);
4263       // If we have changed the HBase state it is better to run hbck again
4264       // to see if we haven't broken something else in the process.
4265       // We run it only once more because otherwise we can easily fall into
4266       // an infinite loop.
4267       if (shouldRerun()) {
4268         try {
4269           LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4270           Thread.sleep(sleepBeforeRerun);
4271         } catch (InterruptedException ie) {
4272           LOG.warn("Interrupted while sleeping");
4273           return this;
4274         }
4275         // Just report
4276         setFixAssignments(false);
4277         setFixMeta(false);
4278         setFixHdfsHoles(false);
4279         setFixHdfsOverlaps(false);
4280         setFixVersionFile(false);
4281         setFixTableOrphans(false);
4282         errors.resetErrors();
4283         code = onlineHbck();
4284         setRetCode(code);
4285       }
4286     } finally {
4287       IOUtils.cleanup(null, connection, meta, admin);
4288     }
4289     return this;
4290   }
4291 
4292   /**
4293    * ls -r for debugging purposes
4294    */
4295   void debugLsr(Path p) throws IOException {
4296     debugLsr(getConf(), p, errors);
4297   }
4298 
4299   /**
4300    * ls -r for debugging purposes
4301    */
4302   public static void debugLsr(Configuration conf,
4303       Path p) throws IOException {
4304     debugLsr(conf, p, new PrintingErrorReporter());
4305   }
4306 
4307   /**
4308    * ls -r for debugging purposes
4309    */
4310   public static void debugLsr(Configuration conf,
4311       Path p, ErrorReporter errors) throws IOException {
4312     if (!LOG.isDebugEnabled() || p == null) {
4313       return;
4314     }
4315     FileSystem fs = p.getFileSystem(conf);
4316 
4317     if (!fs.exists(p)) {
4318       // nothing
4319       return;
4320     }
4321     errors.print(p.toString());
4322 
4323     if (fs.isFile(p)) {
4324       return;
4325     }
4326 
4327     if (fs.getFileStatus(p).isDirectory()) {
4328       FileStatus[] fss= fs.listStatus(p);
4329       for (FileStatus status : fss) {
4330         debugLsr(conf, status.getPath(), errors);
4331       }
4332     }
4333   }
4334 }