View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.FileNotFoundException;
21  import java.io.IOException;
22  import java.io.PrintWriter;
23  import java.io.StringWriter;
24  import java.net.InetAddress;
25  import java.net.URI;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.Collection;
29  import java.util.Collections;
30  import java.util.Comparator;
31  import java.util.HashMap;
32  import java.util.HashSet;
33  import java.util.Iterator;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.Map.Entry;
37  import java.util.Set;
38  import java.util.SortedMap;
39  import java.util.SortedSet;
40  import java.util.TreeMap;
41  import java.util.TreeSet;
42  import java.util.concurrent.Callable;
43  import java.util.concurrent.ConcurrentSkipListMap;
44  import java.util.concurrent.ExecutionException;
45  import java.util.concurrent.ExecutorService;
46  import java.util.concurrent.Future;
47  import java.util.concurrent.ScheduledThreadPoolExecutor;
48  import java.util.concurrent.atomic.AtomicBoolean;
49  import java.util.concurrent.atomic.AtomicInteger;
50  
51  
52  import org.apache.commons.lang.StringUtils;
53  import org.apache.commons.logging.Log;
54  import org.apache.commons.logging.LogFactory;
55  import org.apache.hadoop.hbase.classification.InterfaceAudience;
56  import org.apache.hadoop.hbase.classification.InterfaceStability;
57  import org.apache.hadoop.conf.Configuration;
58  import org.apache.hadoop.conf.Configured;
59  import org.apache.hadoop.fs.FSDataOutputStream;
60  import org.apache.hadoop.fs.FileStatus;
61  import org.apache.hadoop.fs.FileSystem;
62  import org.apache.hadoop.fs.Path;
63  import org.apache.hadoop.fs.permission.FsAction;
64  import org.apache.hadoop.fs.permission.FsPermission;
65  import org.apache.hadoop.hbase.Abortable;
66  import org.apache.hadoop.hbase.Cell;
67  import org.apache.hadoop.hbase.ClusterStatus;
68  import org.apache.hadoop.hbase.HBaseConfiguration;
69  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
70  import org.apache.hadoop.hbase.HColumnDescriptor;
71  import org.apache.hadoop.hbase.HConstants;
72  import org.apache.hadoop.hbase.HRegionInfo;
73  import org.apache.hadoop.hbase.HRegionLocation;
74  import org.apache.hadoop.hbase.HTableDescriptor;
75  import org.apache.hadoop.hbase.KeyValue;
76  import org.apache.hadoop.hbase.MasterNotRunningException;
77  import org.apache.hadoop.hbase.MetaTableAccessor;
78  import org.apache.hadoop.hbase.RegionLocations;
79  import org.apache.hadoop.hbase.ServerName;
80  import org.apache.hadoop.hbase.TableDescriptor;
81  import org.apache.hadoop.hbase.TableName;
82  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
83  import org.apache.hadoop.hbase.classification.InterfaceAudience;
84  import org.apache.hadoop.hbase.classification.InterfaceStability;
85  import org.apache.hadoop.hbase.client.Admin;
86  import org.apache.hadoop.hbase.client.ClusterConnection;
87  import org.apache.hadoop.hbase.client.ConnectionFactory;
88  import org.apache.hadoop.hbase.client.Delete;
89  import org.apache.hadoop.hbase.client.Get;
90  import org.apache.hadoop.hbase.client.HBaseAdmin;
91  import org.apache.hadoop.hbase.client.HConnectable;
92  import org.apache.hadoop.hbase.client.HConnection;
93  import org.apache.hadoop.hbase.client.HConnectionManager;
94  import org.apache.hadoop.hbase.client.MetaScanner;
95  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
96  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
97  import org.apache.hadoop.hbase.client.Put;
98  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
99  import org.apache.hadoop.hbase.client.Result;
100 import org.apache.hadoop.hbase.client.RowMutations;
101 import org.apache.hadoop.hbase.client.Table;
102 import org.apache.hadoop.hbase.client.TableState;
103 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
104 import org.apache.hadoop.hbase.io.hfile.HFile;
105 import org.apache.hadoop.hbase.master.MasterFileSystem;
106 import org.apache.hadoop.hbase.master.RegionState;
107 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
108 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
109 import org.apache.hadoop.hbase.regionserver.HRegion;
110 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
111 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
112 import org.apache.hadoop.hbase.security.AccessDeniedException;
113 import org.apache.hadoop.hbase.security.UserProvider;
114 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
115 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
116 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
117 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
118 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
119 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
120 import org.apache.hadoop.hbase.wal.WALSplitter;
121 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
122 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
123 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
124 import org.apache.hadoop.io.IOUtils;
125 import org.apache.hadoop.ipc.RemoteException;
126 import org.apache.hadoop.security.UserGroupInformation;
127 import org.apache.hadoop.util.ReflectionUtils;
128 import org.apache.hadoop.util.Tool;
129 import org.apache.hadoop.util.ToolRunner;
130 import org.apache.zookeeper.KeeperException;
131 
132 import com.google.common.base.Joiner;
133 import com.google.common.base.Preconditions;
134 import com.google.common.collect.Lists;
135 import com.google.common.collect.Multimap;
136 import com.google.common.collect.TreeMultimap;
137 import com.google.protobuf.ServiceException;
138 
139 /**
140  * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
141  * table integrity problems in a corrupted HBase.
142  * <p>
143  * Region consistency checks verify that hbase:meta, region deployment on region
144  * servers and the state of data in HDFS (.regioninfo files) all are in
145  * accordance.
146  * <p>
147  * Table integrity checks verify that all possible row keys resolve to exactly
148  * one region of a table.  This means there are no individual degenerate
149  * or backwards regions; no holes between regions; and that there are no
150  * overlapping regions.
151  * <p>
152  * The general repair strategy works in two phases:
153  * <ol>
154  * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
155  * <li> Repair Region Consistency with hbase:meta and assignments
156  * </ol>
157  * <p>
158  * For table integrity repairs, the tables' region directories are scanned
159  * for .regioninfo files.  Each table's integrity is then verified.  If there
160  * are any orphan regions (regions with no .regioninfo files) or holes, new
161  * regions are fabricated.  Backwards regions are sidelined as well as empty
162  * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
163  * a new region is created and all data is merged into the new region.
164  * <p>
165  * Table integrity repairs deal solely with HDFS and could potentially be done
166  * offline -- the hbase region servers or master do not need to be running.
167  * This phase can eventually be used to completely reconstruct the hbase:meta table in
168  * an offline fashion.
169  * <p>
170  * Region consistency requires three conditions -- 1) valid .regioninfo file
171  * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
172  * and 3) a region is deployed only at the regionserver that was assigned to
173  * with proper state in the master.
174  * <p>
175  * Region consistency repairs require hbase to be online so that hbck can
176  * contact the HBase master and region servers.  The hbck#connect() method must
177  * first be called successfully.  Much of the region consistency information
178  * is transient and less risky to repair.
179  * <p>
180  * If hbck is run from the command line, there are a handful of arguments that
181  * can be used to limit the kinds of repairs hbck will do.  See the code in
182  * {@link #printUsageAndExit()} for more details.
183  */
184 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
185 @InterfaceStability.Evolving
186 public class HBaseFsck extends Configured {
187   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
188   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
189   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
190   private static boolean rsSupportsOffline = true;
191   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
192   private static final int DEFAULT_MAX_MERGE = 5;
193   private static final String TO_BE_LOADED = "to_be_loaded";
194   private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
195 
196 
197   /**********************
198    * Internal resources
199    **********************/
200   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
201   private ClusterStatus status;
202   private ClusterConnection connection;
203   private Admin admin;
204   private Table meta;
205   // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
206   protected ExecutorService executor;
207   private long startMillis = System.currentTimeMillis();
208   private HFileCorruptionChecker hfcc;
209   private int retcode = 0;
210   private Path HBCK_LOCK_PATH;
211   private FSDataOutputStream hbckOutFd;
212   // This lock is to prevent cleanup of balancer resources twice between
213   // ShutdownHook and the main code. We cleanup only if the connect() is
214   // successful
215   private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
216 
217   /***********
218    * Options
219    ***********/
220   private static boolean details = false; // do we display the full report
221   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
222   private boolean fixAssignments = false; // fix assignment errors?
223   private boolean fixMeta = false; // fix meta errors?
224   private boolean checkHdfs = true; // load and check fs consistency?
225   private boolean fixHdfsHoles = false; // fix fs holes?
226   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
227   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
228   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
229   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
230   private boolean fixSplitParents = false; // fix lingering split parents
231   private boolean fixReferenceFiles = false; // fix lingering reference store file
232   private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
233   private boolean fixTableLocks = false; // fix table locks which are expired
234   private boolean fixAny = false; // Set to true if any of the fix is required.
235 
236   // limit checking/fixes to listed tables, if empty attempt to check/fix all
237   // hbase:meta are always checked
238   private Set<TableName> tablesIncluded = new HashSet<TableName>();
239   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
240   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
241   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
242   private Path sidelineDir = null;
243 
244   private boolean rerun = false; // if we tried to fix something, rerun hbck
245   private static boolean summary = false; // if we want to print less output
246   private boolean checkMetaOnly = false;
247   private boolean checkRegionBoundaries = false;
248   private boolean ignorePreCheckPermission = false; // if pre-check permission
249 
250   /*********
251    * State
252    *********/
253   final private ErrorReporter errors;
254   int fixes = 0;
255 
256   /**
257    * This map contains the state of all hbck items.  It maps from encoded region
258    * name to HbckInfo structure.  The information contained in HbckInfo is used
259    * to detect and correct consistency (hdfs/meta/deployment) problems.
260    */
261   private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
262   private TreeSet<TableName> disabledTables =
263     new TreeSet<TableName>();
264   // Empty regioninfo qualifiers in hbase:meta
265   private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
266 
267   /**
268    * This map from Tablename -> TableInfo contains the structures necessary to
269    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
270    * to prevent dupes.
271    *
272    * If tablesIncluded is empty, this map contains all tables.
273    * Otherwise, it contains only meta tables and tables in tablesIncluded,
274    * unless checkMetaOnly is specified, in which case, it contains only
275    * the meta table
276    */
277   private SortedMap<TableName, TableInfo> tablesInfo =
278       new ConcurrentSkipListMap<TableName, TableInfo>();
279 
280   /**
281    * When initially looking at HDFS, we attempt to find any orphaned data.
282    */
283   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
284 
285   private Map<TableName, Set<String>> orphanTableDirs =
286       new HashMap<TableName, Set<String>>();
287 
288   /**
289    * Constructor
290    *
291    * @param conf Configuration object
292    * @throws MasterNotRunningException if the master is not running
293    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
294    */
295   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
296       ZooKeeperConnectionException, IOException, ClassNotFoundException {
297     super(conf);
298     // make a copy, just to be sure we're not overriding someone else's config
299     setConf(HBaseConfiguration.create(getConf()));
300     // disable blockcache for tool invocation, see HBASE-10500
301     getConf().setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0);
302     errors = getErrorReporter(conf);
303 
304     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
305     executor = new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
306   }
307 
308   /**
309    * Constructor
310    *
311    * @param conf
312    *          Configuration object
313    * @throws MasterNotRunningException
314    *           if the master is not running
315    * @throws ZooKeeperConnectionException
316    *           if unable to connect to ZooKeeper
317    */
318   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
319       ZooKeeperConnectionException, IOException, ClassNotFoundException {
320     super(conf);
321     errors = getErrorReporter(getConf());
322     this.executor = exec;
323   }
324 
325   /**
326    * This method maintains a lock using a file. If the creation fails we return null
327    *
328    * @return FSDataOutputStream object corresponding to the newly opened lock file
329    * @throws IOException
330    */
331   private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
332     try {
333       FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
334       FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
335           HConstants.DATA_FILE_UMASK_KEY);
336       Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
337       fs.mkdirs(tmpDir);
338       HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
339       final FSDataOutputStream out = FSUtils.create(fs, HBCK_LOCK_PATH, defaultPerms, false);
340       out.writeBytes(InetAddress.getLocalHost().toString());
341       out.flush();
342       return out;
343     } catch(RemoteException e) {
344       if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
345         return null;
346       } else {
347         throw e;
348       }
349     }
350   }
351 
352   private void unlockHbck() {
353     if(hbckLockCleanup.compareAndSet(true, false)){
354       IOUtils.closeStream(hbckOutFd);
355       try{
356         FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()), HBCK_LOCK_PATH, true);
357       } catch(IOException ioe) {
358         LOG.warn("Failed to delete " + HBCK_LOCK_PATH);
359         LOG.debug(ioe);
360       }
361     }
362   }
363 
364   /**
365    * To repair region consistency, one must call connect() in order to repair
366    * online state.
367    */
368   public void connect() throws IOException {
369 
370     // Check if another instance of balancer is running
371     hbckOutFd = checkAndMarkRunningHbck();
372     if (hbckOutFd == null) {
373       setRetCode(-1);
374       LOG.error("Another instance of hbck is running, exiting this instance.[If you are sure" +
375           " no other instance is running, delete the lock file " +
376           HBCK_LOCK_PATH + " and rerun the tool]");
377       throw new IOException("Duplicate hbck - Abort");
378     }
379 
380     // Make sure to cleanup the lock
381     hbckLockCleanup.set(true);
382 
383     // Add a shutdown hook to this thread, incase user tries to
384     // kill the hbck with a ctrl-c, we want to cleanup the lock so that
385     // it is available for further calls
386     Runtime.getRuntime().addShutdownHook(new Thread() {
387       @Override
388       public void run() {
389           unlockHbck();
390       }
391     });
392     LOG.debug("Launching hbck");
393 
394     connection = (ClusterConnection)ConnectionFactory.createConnection(getConf());
395     admin = connection.getAdmin();
396     meta = connection.getTable(TableName.META_TABLE_NAME);
397     status = admin.getClusterStatus();
398   }
399 
400   /**
401    * Get deployed regions according to the region servers.
402    */
403   private void loadDeployedRegions() throws IOException, InterruptedException {
404     // From the master, get a list of all known live region servers
405     Collection<ServerName> regionServers = status.getServers();
406     errors.print("Number of live region servers: " + regionServers.size());
407     if (details) {
408       for (ServerName rsinfo: regionServers) {
409         errors.print("  " + rsinfo.getServerName());
410       }
411     }
412 
413     // From the master, get a list of all dead region servers
414     Collection<ServerName> deadRegionServers = status.getDeadServerNames();
415     errors.print("Number of dead region servers: " + deadRegionServers.size());
416     if (details) {
417       for (ServerName name: deadRegionServers) {
418         errors.print("  " + name);
419       }
420     }
421 
422     // Print the current master name and state
423     errors.print("Master: " + status.getMaster());
424 
425     // Print the list of all backup masters
426     Collection<ServerName> backupMasters = status.getBackupMasters();
427     errors.print("Number of backup masters: " + backupMasters.size());
428     if (details) {
429       for (ServerName name: backupMasters) {
430         errors.print("  " + name);
431       }
432     }
433 
434     errors.print("Average load: " + status.getAverageLoad());
435     errors.print("Number of requests: " + status.getRequestsCount());
436     errors.print("Number of regions: " + status.getRegionsCount());
437 
438     Map<String, RegionState> rits = status.getRegionsInTransition();
439     errors.print("Number of regions in transition: " + rits.size());
440     if (details) {
441       for (RegionState state: rits.values()) {
442         errors.print("  " + state.toDescriptiveString());
443       }
444     }
445 
446     // Determine what's deployed
447     processRegionServers(regionServers);
448   }
449 
450   /**
451    * Clear the current state of hbck.
452    */
453   private void clearState() {
454     // Make sure regionInfo is empty before starting
455     fixes = 0;
456     regionInfoMap.clear();
457     emptyRegionInfoQualifiers.clear();
458     disabledTables.clear();
459     errors.clear();
460     tablesInfo.clear();
461     orphanHdfsDirs.clear();
462   }
463 
464   /**
465    * This repair method analyzes hbase data in hdfs and repairs it to satisfy
466    * the table integrity rules.  HBase doesn't need to be online for this
467    * operation to work.
468    */
469   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
470     // Initial pass to fix orphans.
471     if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
472         || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
473       LOG.info("Loading regioninfos HDFS");
474       // if nothing is happening this should always complete in two iterations.
475       int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
476       int curIter = 0;
477       do {
478         clearState(); // clears hbck state and reset fixes to 0 and.
479         // repair what's on HDFS
480         restoreHdfsIntegrity();
481         curIter++;// limit the number of iterations.
482       } while (fixes > 0 && curIter <= maxIterations);
483 
484       // Repairs should be done in the first iteration and verification in the second.
485       // If there are more than 2 passes, something funny has happened.
486       if (curIter > 2) {
487         if (curIter == maxIterations) {
488           LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
489               + "Tables integrity may not be fully repaired!");
490         } else {
491           LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
492         }
493       }
494     }
495   }
496 
497   /**
498    * This repair method requires the cluster to be online since it contacts
499    * region servers and the masters.  It makes each region's state in HDFS, in
500    * hbase:meta, and deployments consistent.
501    *
502    * @return If > 0 , number of errors detected, if < 0 there was an unrecoverable
503    * error.  If 0, we have a clean hbase.
504    */
505   public int onlineConsistencyRepair() throws IOException, KeeperException,
506     InterruptedException {
507     clearState();
508 
509     // get regions according to what is online on each RegionServer
510     loadDeployedRegions();
511     // check whether hbase:meta is deployed and online
512     recordMetaRegion();
513     // Check if hbase:meta is found only once and in the right place
514     if (!checkMetaRegion()) {
515       String errorMsg = "hbase:meta table is not consistent. ";
516       if (shouldFixAssignments()) {
517         errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
518       } else {
519         errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
520       }
521       errors.reportError(errorMsg + " Exiting...");
522       return -2;
523     }
524     // Not going with further consistency check for tables when hbase:meta itself is not consistent.
525     LOG.info("Loading regionsinfo from the hbase:meta table");
526     boolean success = loadMetaEntries();
527     if (!success) return -1;
528 
529     // Empty cells in hbase:meta?
530     reportEmptyMetaCells();
531 
532     // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
533     if (shouldFixEmptyMetaCells()) {
534       fixEmptyMetaCells();
535     }
536 
537     // get a list of all tables that have not changed recently.
538     if (!checkMetaOnly) {
539       reportTablesInFlux();
540     }
541 
542     // load regiondirs and regioninfos from HDFS
543     if (shouldCheckHdfs()) {
544       loadHdfsRegionDirs();
545       loadHdfsRegionInfos();
546     }
547 
548     // Get disabled tables from ZooKeeper
549     loadDisabledTables();
550 
551     // fix the orphan tables
552     fixOrphanTables();
553 
554     // Check and fix consistency
555     checkAndFixConsistency();
556 
557     // Check integrity (does not fix)
558     checkIntegrity();
559     return errors.getErrorList().size();
560   }
561 
562   /**
563    * Contacts the master and prints out cluster-wide information
564    * @return 0 on success, non-zero on failure
565    */
566   public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
567     // print hbase server version
568     errors.print("Version: " + status.getHBaseVersion());
569     offlineHdfsIntegrityRepair();
570 
571     // turn the balancer off
572     boolean oldBalancer = admin.setBalancerRunning(false, true);
573     try {
574       onlineConsistencyRepair();
575     }
576     finally {
577       admin.setBalancerRunning(oldBalancer, false);
578     }
579 
580     if (checkRegionBoundaries) {
581       checkRegionBoundaries();
582     }
583 
584     offlineReferenceFileRepair();
585 
586     checkAndFixTableLocks();
587 
588     // Remove the hbck lock
589     unlockHbck();
590 
591     // Print table summary
592     printTableSummary(tablesInfo);
593     return errors.summarize();
594   }
595 
596   public static byte[] keyOnly (byte[] b) {
597     if (b == null)
598       return b;
599     int rowlength = Bytes.toShort(b, 0);
600     byte[] result = new byte[rowlength];
601     System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
602     return result;
603   }
604 
605   private static class RegionBoundariesInformation {
606     public byte [] regionName;
607     public byte [] metaFirstKey;
608     public byte [] metaLastKey;
609     public byte [] storesFirstKey;
610     public byte [] storesLastKey;
611     @Override
612     public String toString () {
613       return "regionName=" + Bytes.toStringBinary(regionName) +
614              "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
615              "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
616              "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
617              "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
618     }
619   }
620 
621   public void checkRegionBoundaries() {
622     try {
623       ByteArrayComparator comparator = new ByteArrayComparator();
624       List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), false);
625       final RegionBoundariesInformation currentRegionBoundariesInformation =
626           new RegionBoundariesInformation();
627       Path hbaseRoot = FSUtils.getRootDir(getConf());
628       for (HRegionInfo regionInfo : regions) {
629         Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
630         currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
631         // For each region, get the start and stop key from the META and compare them to the
632         // same information from the Stores.
633         Path path = new Path(tableDir, regionInfo.getEncodedName());
634         FileSystem fs = path.getFileSystem(getConf());
635         FileStatus[] files = fs.listStatus(path);
636         // For all the column families in this region...
637         byte[] storeFirstKey = null;
638         byte[] storeLastKey = null;
639         for (FileStatus file : files) {
640           String fileName = file.getPath().toString();
641           fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
642           if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
643             FileStatus[] storeFiles = fs.listStatus(file.getPath());
644             // For all the stores in this column family.
645             for (FileStatus storeFile : storeFiles) {
646               HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
647                   getConf()), getConf());
648               if ((reader.getFirstKey() != null)
649                   && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
650                       reader.getFirstKey()) > 0))) {
651                 storeFirstKey = reader.getFirstKey();
652               }
653               if ((reader.getLastKey() != null)
654                   && ((storeLastKey == null) || (comparator.compare(storeLastKey,
655                       reader.getLastKey())) < 0)) {
656                 storeLastKey = reader.getLastKey();
657               }
658               reader.close();
659             }
660           }
661         }
662         currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
663         currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
664         currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
665         currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
666         if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
667           currentRegionBoundariesInformation.metaFirstKey = null;
668         if (currentRegionBoundariesInformation.metaLastKey.length == 0)
669           currentRegionBoundariesInformation.metaLastKey = null;
670 
671         // For a region to be correct, we need the META start key to be smaller or equal to the
672         // smallest start key from all the stores, and the start key from the next META entry to
673         // be bigger than the last key from all the current stores. First region start key is null;
674         // Last region end key is null; some regions can be empty and not have any store.
675 
676         boolean valid = true;
677         // Checking start key.
678         if ((currentRegionBoundariesInformation.storesFirstKey != null)
679             && (currentRegionBoundariesInformation.metaFirstKey != null)) {
680           valid = valid
681               && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
682                 currentRegionBoundariesInformation.metaFirstKey) >= 0;
683         }
684         // Checking stop key.
685         if ((currentRegionBoundariesInformation.storesLastKey != null)
686             && (currentRegionBoundariesInformation.metaLastKey != null)) {
687           valid = valid
688               && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
689                 currentRegionBoundariesInformation.metaLastKey) < 0;
690         }
691         if (!valid) {
692           errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
693             tablesInfo.get(regionInfo.getTable()));
694           LOG.warn("Region's boundaries not alligned between stores and META for:");
695           LOG.warn(currentRegionBoundariesInformation);
696         }
697       }
698     } catch (IOException e) {
699       LOG.error(e);
700     }
701   }
702 
703   /**
704    * Iterates through the list of all orphan/invalid regiondirs.
705    */
706   private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
707     for (HbckInfo hi : orphanHdfsDirs) {
708       LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
709       adoptHdfsOrphan(hi);
710     }
711   }
712 
713   /**
714    * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
715    * these orphans by creating a new region, and moving the column families,
716    * recovered edits, WALs, into the new region dir.  We determine the region
717    * startkey and endkeys by looking at all of the hfiles inside the column
718    * families to identify the min and max keys. The resulting region will
719    * likely violate table integrity but will be dealt with by merging
720    * overlapping regions.
721    */
722   @SuppressWarnings("deprecation")
723   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
724     Path p = hi.getHdfsRegionDir();
725     FileSystem fs = p.getFileSystem(getConf());
726     FileStatus[] dirs = fs.listStatus(p);
727     if (dirs == null) {
728       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
729           p + ". This dir could probably be deleted.");
730       return ;
731     }
732 
733     TableName tableName = hi.getTableName();
734     TableInfo tableInfo = tablesInfo.get(tableName);
735     Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
736     HTableDescriptor template = tableInfo.getHTD();
737 
738     // find min and max key values
739     Pair<byte[],byte[]> orphanRegionRange = null;
740     for (FileStatus cf : dirs) {
741       String cfName= cf.getPath().getName();
742       // TODO Figure out what the special dirs are
743       if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
744 
745       FileStatus[] hfiles = fs.listStatus(cf.getPath());
746       for (FileStatus hfile : hfiles) {
747         byte[] start, end;
748         HFile.Reader hf = null;
749         try {
750           CacheConfig cacheConf = new CacheConfig(getConf());
751           hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
752           hf.loadFileInfo();
753           KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
754           start = startKv.getRow();
755           KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
756           end = endKv.getRow();
757         } catch (IOException ioe) {
758           LOG.warn("Problem reading orphan file " + hfile + ", skipping");
759           continue;
760         } catch (NullPointerException ioe) {
761           LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
762           continue;
763         } finally {
764           if (hf != null) {
765             hf.close();
766           }
767         }
768 
769         // expand the range to include the range of all hfiles
770         if (orphanRegionRange == null) {
771           // first range
772           orphanRegionRange = new Pair<byte[], byte[]>(start, end);
773         } else {
774           // TODO add test
775 
776           // expand range only if the hfile is wider.
777           if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
778             orphanRegionRange.setFirst(start);
779           }
780           if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
781             orphanRegionRange.setSecond(end);
782           }
783         }
784       }
785     }
786     if (orphanRegionRange == null) {
787       LOG.warn("No data in dir " + p + ", sidelining data");
788       fixes++;
789       sidelineRegionDir(fs, hi);
790       return;
791     }
792     LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
793         Bytes.toString(orphanRegionRange.getSecond()) + ")");
794 
795     // create new region on hdfs.  move data into place.
796     HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond());
797     LOG.info("Creating new region : " + hri);
798     HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
799     Path target = region.getRegionFileSystem().getRegionDir();
800 
801     // rename all the data to new region
802     mergeRegionDirs(target, hi);
803     fixes++;
804   }
805 
806   /**
807    * This method determines if there are table integrity errors in HDFS.  If
808    * there are errors and the appropriate "fix" options are enabled, the method
809    * will first correct orphan regions making them into legit regiondirs, and
810    * then reload to merge potentially overlapping regions.
811    *
812    * @return number of table integrity errors found
813    */
814   private int restoreHdfsIntegrity() throws IOException, InterruptedException {
815     // Determine what's on HDFS
816     LOG.info("Loading HBase regioninfo from HDFS...");
817     loadHdfsRegionDirs(); // populating regioninfo table.
818 
819     int errs = errors.getErrorList().size();
820     // First time just get suggestions.
821     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
822     checkHdfsIntegrity(false, false);
823 
824     if (errors.getErrorList().size() == errs) {
825       LOG.info("No integrity errors.  We are done with this phase. Glorious.");
826       return 0;
827     }
828 
829     if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
830       adoptHdfsOrphans(orphanHdfsDirs);
831       // TODO optimize by incrementally adding instead of reloading.
832     }
833 
834     // Make sure there are no holes now.
835     if (shouldFixHdfsHoles()) {
836       clearState(); // this also resets # fixes.
837       loadHdfsRegionDirs();
838       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
839       tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
840     }
841 
842     // Now we fix overlaps
843     if (shouldFixHdfsOverlaps()) {
844       // second pass we fix overlaps.
845       clearState(); // this also resets # fixes.
846       loadHdfsRegionDirs();
847       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
848       tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
849     }
850 
851     return errors.getErrorList().size();
852   }
853 
854   /**
855    * Scan all the store file names to find any lingering reference files,
856    * which refer to some none-exiting files. If "fix" option is enabled,
857    * any lingering reference file will be sidelined if found.
858    * <p>
859    * Lingering reference file prevents a region from opening. It has to
860    * be fixed before a cluster can start properly.
861    */
862   private void offlineReferenceFileRepair() throws IOException {
863     Configuration conf = getConf();
864     Path hbaseRoot = FSUtils.getRootDir(conf);
865     FileSystem fs = hbaseRoot.getFileSystem(conf);
866     Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot);
867     for (Path path: allFiles.values()) {
868       boolean isReference = false;
869       try {
870         isReference = StoreFileInfo.isReference(path);
871       } catch (Throwable t) {
872         // Ignore. Some files may not be store files at all.
873         // For example, files under .oldlogs folder in hbase:meta
874         // Warning message is already logged by
875         // StoreFile#isReference.
876       }
877       if (!isReference) continue;
878 
879       Path referredToFile = StoreFileInfo.getReferredToFile(path);
880       if (fs.exists(referredToFile)) continue;  // good, expected
881 
882       // Found a lingering reference file
883       errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
884         "Found lingering reference file " + path);
885       if (!shouldFixReferenceFiles()) continue;
886 
887       // Now, trying to fix it since requested
888       boolean success = false;
889       String pathStr = path.toString();
890 
891       // A reference file path should be like
892       // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
893       // Up 5 directories to get the root folder.
894       // So the file will be sidelined to a similar folder structure.
895       int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
896       for (int i = 0; index > 0 && i < 5; i++) {
897         index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
898       }
899       if (index > 0) {
900         Path rootDir = getSidelineDir();
901         Path dst = new Path(rootDir, pathStr.substring(index + 1));
902         fs.mkdirs(dst.getParent());
903         LOG.info("Trying to sildeline reference file "
904           + path + " to " + dst);
905         setShouldRerun();
906 
907         success = fs.rename(path, dst);
908       }
909       if (!success) {
910         LOG.error("Failed to sideline reference file " + path);
911       }
912     }
913   }
914 
915   /**
916    * TODO -- need to add tests for this.
917    */
918   private void reportEmptyMetaCells() {
919     errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
920       emptyRegionInfoQualifiers.size());
921     if (details) {
922       for (Result r: emptyRegionInfoQualifiers) {
923         errors.print("  " + r);
924       }
925     }
926   }
927 
928   /**
929    * TODO -- need to add tests for this.
930    */
931   private void reportTablesInFlux() {
932     AtomicInteger numSkipped = new AtomicInteger(0);
933     HTableDescriptor[] allTables = getTables(numSkipped);
934     errors.print("Number of Tables: " + allTables.length);
935     if (details) {
936       if (numSkipped.get() > 0) {
937         errors.detail("Number of Tables in flux: " + numSkipped.get());
938       }
939       for (HTableDescriptor td : allTables) {
940         errors.detail("  Table: " + td.getTableName() + "\t" +
941                            (td.isReadOnly() ? "ro" : "rw") + "\t" +
942                             (td.isMetaRegion() ? "META" : "    ") + "\t" +
943                            " families: " + td.getFamilies().size());
944       }
945     }
946   }
947 
948   public ErrorReporter getErrors() {
949     return errors;
950   }
951 
952   /**
953    * Read the .regioninfo file from the file system.  If there is no
954    * .regioninfo, add it to the orphan hdfs region list.
955    */
956   private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
957     Path regionDir = hbi.getHdfsRegionDir();
958     if (regionDir == null) {
959       LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
960       return;
961     }
962 
963     if (hbi.hdfsEntry.hri != null) {
964       // already loaded data
965       return;
966     }
967 
968     FileSystem fs = FileSystem.get(getConf());
969     HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
970     LOG.debug("HRegionInfo read: " + hri.toString());
971     hbi.hdfsEntry.hri = hri;
972   }
973 
974   /**
975    * Exception thrown when a integrity repair operation fails in an
976    * unresolvable way.
977    */
978   public static class RegionRepairException extends IOException {
979     private static final long serialVersionUID = 1L;
980     final IOException ioe;
981     public RegionRepairException(String s, IOException ioe) {
982       super(s);
983       this.ioe = ioe;
984     }
985   }
986 
987   /**
988    * Populate hbi's from regionInfos loaded from file system.
989    */
990   private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
991       throws IOException, InterruptedException {
992     tablesInfo.clear(); // regenerating the data
993     // generate region split structure
994     Collection<HbckInfo> hbckInfos = regionInfoMap.values();
995 
996     // Parallelized read of .regioninfo files.
997     List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
998     List<Future<Void>> hbiFutures;
999 
1000     for (HbckInfo hbi : hbckInfos) {
1001       WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
1002       hbis.add(work);
1003     }
1004 
1005     // Submit and wait for completion
1006     hbiFutures = executor.invokeAll(hbis);
1007 
1008     for(int i=0; i<hbiFutures.size(); i++) {
1009       WorkItemHdfsRegionInfo work = hbis.get(i);
1010       Future<Void> f = hbiFutures.get(i);
1011       try {
1012         f.get();
1013       } catch(ExecutionException e) {
1014         LOG.warn("Failed to read .regioninfo file for region " +
1015               work.hbi.getRegionNameAsString(), e.getCause());
1016       }
1017     }
1018 
1019     Path hbaseRoot = FSUtils.getRootDir(getConf());
1020     FileSystem fs = hbaseRoot.getFileSystem(getConf());
1021     // serialized table info gathering.
1022     for (HbckInfo hbi: hbckInfos) {
1023 
1024       if (hbi.getHdfsHRI() == null) {
1025         // was an orphan
1026         continue;
1027       }
1028 
1029 
1030       // get table name from hdfs, populate various HBaseFsck tables.
1031       TableName tableName = hbi.getTableName();
1032       if (tableName == null) {
1033         // There was an entry in hbase:meta not in the HDFS?
1034         LOG.warn("tableName was null for: " + hbi);
1035         continue;
1036       }
1037 
1038       TableInfo modTInfo = tablesInfo.get(tableName);
1039       if (modTInfo == null) {
1040         // only executed once per table.
1041         modTInfo = new TableInfo(tableName);
1042         tablesInfo.put(tableName, modTInfo);
1043         try {
1044           TableDescriptor htd =
1045               FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
1046           modTInfo.htds.add(htd.getHTableDescriptor());
1047         } catch (IOException ioe) {
1048           if (!orphanTableDirs.containsKey(tableName)) {
1049             LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
1050             //should only report once for each table
1051             errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
1052                 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
1053             Set<String> columns = new HashSet<String>();
1054             orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
1055           }
1056         }
1057       }
1058       if (!hbi.isSkipChecks()) {
1059         modTInfo.addRegionInfo(hbi);
1060       }
1061     }
1062 
1063     loadTableInfosForTablesWithNoRegion();
1064 
1065     return tablesInfo;
1066   }
1067 
1068   /**
1069    * To get the column family list according to the column family dirs
1070    * @param columns
1071    * @param hbi
1072    * @return a set of column families
1073    * @throws IOException
1074    */
1075   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
1076     Path regionDir = hbi.getHdfsRegionDir();
1077     FileSystem fs = regionDir.getFileSystem(getConf());
1078     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
1079     for (FileStatus subdir : subDirs) {
1080       String columnfamily = subdir.getPath().getName();
1081       columns.add(columnfamily);
1082     }
1083     return columns;
1084   }
1085 
1086   /**
1087    * To fabricate a .tableinfo file with following contents<br>
1088    * 1. the correct tablename <br>
1089    * 2. the correct colfamily list<br>
1090    * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1091    * @throws IOException
1092    */
1093   private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1094       Set<String> columns) throws IOException {
1095     if (columns ==null || columns.isEmpty()) return false;
1096     HTableDescriptor htd = new HTableDescriptor(tableName);
1097     for (String columnfamimly : columns) {
1098       htd.addFamily(new HColumnDescriptor(columnfamimly));
1099     }
1100     fstd.createTableDescriptor(new TableDescriptor(htd, TableState.State.ENABLED), true);
1101     return true;
1102   }
1103 
1104   /**
1105    * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1106    * @throws IOException
1107    */
1108   public void fixEmptyMetaCells() throws IOException {
1109     if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1110       LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1111       for (Result region : emptyRegionInfoQualifiers) {
1112         deleteMetaRegion(region.getRow());
1113         errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1114       }
1115       emptyRegionInfoQualifiers.clear();
1116     }
1117   }
1118 
1119   /**
1120    * To fix orphan table by creating a .tableinfo file under tableDir <br>
1121    * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1122    * 2. else create a default .tableinfo file with following items<br>
1123    * &nbsp;2.1 the correct tablename <br>
1124    * &nbsp;2.2 the correct colfamily list<br>
1125    * &nbsp;2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1126    * @throws IOException
1127    */
1128   public void fixOrphanTables() throws IOException {
1129     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1130 
1131       List<TableName> tmpList = new ArrayList<TableName>();
1132       tmpList.addAll(orphanTableDirs.keySet());
1133       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1134       Iterator<Entry<TableName, Set<String>>> iter =
1135           orphanTableDirs.entrySet().iterator();
1136       int j = 0;
1137       int numFailedCase = 0;
1138       FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1139       while (iter.hasNext()) {
1140         Entry<TableName, Set<String>> entry =
1141             iter.next();
1142         TableName tableName = entry.getKey();
1143         LOG.info("Trying to fix orphan table error: " + tableName);
1144         if (j < htds.length) {
1145           if (tableName.equals(htds[j].getTableName())) {
1146             HTableDescriptor htd = htds[j];
1147             LOG.info("fixing orphan table: " + tableName + " from cache");
1148             fstd.createTableDescriptor(new TableDescriptor(htd, TableState.State.ENABLED), true);
1149             j++;
1150             iter.remove();
1151           }
1152         } else {
1153           if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1154             LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1155             LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1156             iter.remove();
1157           } else {
1158             LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1159             numFailedCase++;
1160           }
1161         }
1162         fixes++;
1163       }
1164 
1165       if (orphanTableDirs.isEmpty()) {
1166         // all orphanTableDirs are luckily recovered
1167         // re-run doFsck after recovering the .tableinfo file
1168         setShouldRerun();
1169         LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1170       } else if (numFailedCase > 0) {
1171         LOG.error("Failed to fix " + numFailedCase
1172             + " OrphanTables with default .tableinfo files");
1173       }
1174 
1175     }
1176     //cleanup the list
1177     orphanTableDirs.clear();
1178 
1179   }
1180 
1181   /**
1182    * This borrows code from MasterFileSystem.bootstrap()
1183    *
1184    * @return an open hbase:meta HRegion
1185    */
1186   private HRegion createNewMeta() throws IOException {
1187       Path rootdir = FSUtils.getRootDir(getConf());
1188     Configuration c = getConf();
1189     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1190     HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
1191     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false);
1192     HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor);
1193     MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true);
1194     return meta;
1195   }
1196 
1197   /**
1198    * Generate set of puts to add to new meta.  This expects the tables to be
1199    * clean with no overlaps or holes.  If there are any problems it returns null.
1200    *
1201    * @return An array list of puts to do in bulk, null if tables have problems
1202    */
1203   private ArrayList<Put> generatePuts(
1204       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1205     ArrayList<Put> puts = new ArrayList<Put>();
1206     boolean hasProblems = false;
1207     for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1208       TableName name = e.getKey();
1209 
1210       // skip "hbase:meta"
1211       if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1212         continue;
1213       }
1214 
1215       TableInfo ti = e.getValue();
1216       for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1217           .entrySet()) {
1218         Collection<HbckInfo> his = spl.getValue();
1219         int sz = his.size();
1220         if (sz != 1) {
1221           // problem
1222           LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1223               + " had " +  sz + " regions instead of exactly 1." );
1224           hasProblems = true;
1225           continue;
1226         }
1227 
1228         // add the row directly to meta.
1229         HbckInfo hi = his.iterator().next();
1230         HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
1231         Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1232         puts.add(p);
1233       }
1234     }
1235     return hasProblems ? null : puts;
1236   }
1237 
1238   /**
1239    * Suggest fixes for each table
1240    */
1241   private void suggestFixes(
1242       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1243     for (TableInfo tInfo : tablesInfo.values()) {
1244       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1245       tInfo.checkRegionChain(handler);
1246     }
1247   }
1248 
1249   /**
1250    * Rebuilds meta from information in hdfs/fs.  Depends on configuration
1251    * settings passed into hbck constructor to point to a particular fs/dir.
1252    *
1253    * @param fix flag that determines if method should attempt to fix holes
1254    * @return true if successful, false if attempt failed.
1255    */
1256   public boolean rebuildMeta(boolean fix) throws IOException,
1257       InterruptedException {
1258 
1259     // TODO check to make sure hbase is offline. (or at least the table
1260     // currently being worked on is off line)
1261 
1262     // Determine what's on HDFS
1263     LOG.info("Loading HBase regioninfo from HDFS...");
1264     loadHdfsRegionDirs(); // populating regioninfo table.
1265 
1266     int errs = errors.getErrorList().size();
1267     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1268     checkHdfsIntegrity(false, false);
1269 
1270     // make sure ok.
1271     if (errors.getErrorList().size() != errs) {
1272       // While in error state, iterate until no more fixes possible
1273       while(true) {
1274         fixes = 0;
1275         suggestFixes(tablesInfo);
1276         errors.clear();
1277         loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1278         checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1279 
1280         int errCount = errors.getErrorList().size();
1281 
1282         if (fixes == 0) {
1283           if (errCount > 0) {
1284             return false; // failed to fix problems.
1285           } else {
1286             break; // no fixes and no problems? drop out and fix stuff!
1287           }
1288         }
1289       }
1290     }
1291 
1292     // we can rebuild, move old meta out of the way and start
1293     LOG.info("HDFS regioninfo's seems good.  Sidelining old hbase:meta");
1294     Path backupDir = sidelineOldMeta();
1295 
1296     LOG.info("Creating new hbase:meta");
1297     HRegion meta = createNewMeta();
1298 
1299     // populate meta
1300     List<Put> puts = generatePuts(tablesInfo);
1301     if (puts == null) {
1302       LOG.fatal("Problem encountered when creating new hbase:meta entries.  " +
1303         "You may need to restore the previously sidelined hbase:meta");
1304       return false;
1305     }
1306     meta.batchMutate(puts.toArray(new Put[puts.size()]));
1307     HRegion.closeHRegion(meta);
1308     LOG.info("Success! hbase:meta table rebuilt.");
1309     LOG.info("Old hbase:meta is moved into " + backupDir);
1310     return true;
1311   }
1312 
1313   private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1314       boolean fixOverlaps) throws IOException {
1315     LOG.info("Checking HBase region split map from HDFS data...");
1316     for (TableInfo tInfo : tablesInfo.values()) {
1317       TableIntegrityErrorHandler handler;
1318       if (fixHoles || fixOverlaps) {
1319         handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1320           fixHoles, fixOverlaps);
1321       } else {
1322         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1323       }
1324       if (!tInfo.checkRegionChain(handler)) {
1325         // should dump info as well.
1326         errors.report("Found inconsistency in table " + tInfo.getName());
1327       }
1328     }
1329     return tablesInfo;
1330   }
1331 
1332   private Path getSidelineDir() throws IOException {
1333     if (sidelineDir == null) {
1334       Path hbaseDir = FSUtils.getRootDir(getConf());
1335       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1336       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1337           + startMillis);
1338     }
1339     return sidelineDir;
1340   }
1341 
1342   /**
1343    * Sideline a region dir (instead of deleting it)
1344    */
1345   Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1346     return sidelineRegionDir(fs, null, hi);
1347   }
1348 
1349   /**
1350    * Sideline a region dir (instead of deleting it)
1351    *
1352    * @param parentDir if specified, the region will be sidelined to
1353    * folder like .../parentDir/<table name>/<region name>. The purpose
1354    * is to group together similar regions sidelined, for example, those
1355    * regions should be bulk loaded back later on. If null, it is ignored.
1356    */
1357   Path sidelineRegionDir(FileSystem fs,
1358       String parentDir, HbckInfo hi) throws IOException {
1359     TableName tableName = hi.getTableName();
1360     Path regionDir = hi.getHdfsRegionDir();
1361 
1362     if (!fs.exists(regionDir)) {
1363       LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1364       return null;
1365     }
1366 
1367     Path rootDir = getSidelineDir();
1368     if (parentDir != null) {
1369       rootDir = new Path(rootDir, parentDir);
1370     }
1371     Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1372     Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1373     fs.mkdirs(sidelineRegionDir);
1374     boolean success = false;
1375     FileStatus[] cfs =  fs.listStatus(regionDir);
1376     if (cfs == null) {
1377       LOG.info("Region dir is empty: " + regionDir);
1378     } else {
1379       for (FileStatus cf : cfs) {
1380         Path src = cf.getPath();
1381         Path dst =  new Path(sidelineRegionDir, src.getName());
1382         if (fs.isFile(src)) {
1383           // simple file
1384           success = fs.rename(src, dst);
1385           if (!success) {
1386             String msg = "Unable to rename file " + src +  " to " + dst;
1387             LOG.error(msg);
1388             throw new IOException(msg);
1389           }
1390           continue;
1391         }
1392 
1393         // is a directory.
1394         fs.mkdirs(dst);
1395 
1396         LOG.info("Sidelining files from " + src + " into containing region " + dst);
1397         // FileSystem.rename is inconsistent with directories -- if the
1398         // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1399         // it moves the src into the dst dir resulting in (foo/a/b).  If
1400         // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1401         FileStatus[] hfiles = fs.listStatus(src);
1402         if (hfiles != null && hfiles.length > 0) {
1403           for (FileStatus hfile : hfiles) {
1404             success = fs.rename(hfile.getPath(), dst);
1405             if (!success) {
1406               String msg = "Unable to rename file " + src +  " to " + dst;
1407               LOG.error(msg);
1408               throw new IOException(msg);
1409             }
1410           }
1411         }
1412         LOG.debug("Sideline directory contents:");
1413         debugLsr(sidelineRegionDir);
1414       }
1415     }
1416 
1417     LOG.info("Removing old region dir: " + regionDir);
1418     success = fs.delete(regionDir, true);
1419     if (!success) {
1420       String msg = "Unable to delete dir " + regionDir;
1421       LOG.error(msg);
1422       throw new IOException(msg);
1423     }
1424     return sidelineRegionDir;
1425   }
1426 
1427   /**
1428    * Side line an entire table.
1429    */
1430   void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1431       Path backupHbaseDir) throws IOException {
1432     Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1433     if (fs.exists(tableDir)) {
1434       Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1435       fs.mkdirs(backupTableDir.getParent());
1436       boolean success = fs.rename(tableDir, backupTableDir);
1437       if (!success) {
1438         throw new IOException("Failed to move  " + tableName + " from "
1439             +  tableDir + " to " + backupTableDir);
1440       }
1441     } else {
1442       LOG.info("No previous " + tableName +  " exists.  Continuing.");
1443     }
1444   }
1445 
1446   /**
1447    * @return Path to backup of original directory
1448    */
1449   Path sidelineOldMeta() throws IOException {
1450     // put current hbase:meta aside.
1451     Path hbaseDir = FSUtils.getRootDir(getConf());
1452     FileSystem fs = hbaseDir.getFileSystem(getConf());
1453     Path backupDir = getSidelineDir();
1454     fs.mkdirs(backupDir);
1455 
1456     try {
1457       sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1458     } catch (IOException e) {
1459         LOG.fatal("... failed to sideline meta. Currently in inconsistent state.  To restore "
1460             + "try to rename hbase:meta in " + backupDir.getName() + " to "
1461             + hbaseDir.getName() + ".", e);
1462       throw e; // throw original exception
1463     }
1464     return backupDir;
1465   }
1466 
1467   /**
1468    * Load the list of disabled tables in ZK into local set.
1469    * @throws ZooKeeperConnectionException
1470    * @throws IOException
1471    */
1472   private void loadDisabledTables()
1473   throws IOException {
1474     HConnectionManager.execute(new HConnectable<Void>(getConf()) {
1475       @Override
1476       public Void connect(HConnection connection) throws IOException {
1477         TableName[] tables = connection.listTableNames();
1478         for (TableName table : tables) {
1479           if (connection.getTableState(table)
1480               .inStates(TableState.State.DISABLED, TableState.State.DISABLING)) {
1481             disabledTables.add(table);
1482           }
1483         }
1484         return null;
1485       }
1486     });
1487   }
1488 
1489   /**
1490    * Check if the specified region's table is disabled.
1491    */
1492   private boolean isTableDisabled(HRegionInfo regionInfo) {
1493     return disabledTables.contains(regionInfo.getTable());
1494   }
1495 
1496   /**
1497    * Scan HDFS for all regions, recording their information into
1498    * regionInfoMap
1499    */
1500   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1501     Path rootDir = FSUtils.getRootDir(getConf());
1502     FileSystem fs = rootDir.getFileSystem(getConf());
1503 
1504     // list all tables from HDFS
1505     List<FileStatus> tableDirs = Lists.newArrayList();
1506 
1507     boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1508 
1509     List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1510     for (Path path : paths) {
1511       TableName tableName = FSUtils.getTableName(path);
1512        if ((!checkMetaOnly &&
1513            isTableIncluded(tableName)) ||
1514            tableName.equals(TableName.META_TABLE_NAME)) {
1515          tableDirs.add(fs.getFileStatus(path));
1516        }
1517     }
1518 
1519     // verify that version file exists
1520     if (!foundVersionFile) {
1521       errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1522           "Version file does not exist in root dir " + rootDir);
1523       if (shouldFixVersionFile()) {
1524         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1525             + " file.");
1526         setShouldRerun();
1527         FSUtils.setVersion(fs, rootDir, getConf().getInt(
1528             HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1529             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1530             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1531       }
1532     }
1533 
1534     // level 1:  <HBASE_DIR>/*
1535     List<WorkItemHdfsDir> dirs = new ArrayList<WorkItemHdfsDir>(tableDirs.size());
1536     List<Future<Void>> dirsFutures;
1537 
1538     for (FileStatus tableDir : tableDirs) {
1539       LOG.debug("Loading region dirs from " +tableDir.getPath());
1540       dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir));
1541     }
1542 
1543     // Invoke and wait for Callables to complete
1544     dirsFutures = executor.invokeAll(dirs);
1545 
1546     for(Future<Void> f: dirsFutures) {
1547       try {
1548         f.get();
1549       } catch(ExecutionException e) {
1550         LOG.warn("Could not load region dir " , e.getCause());
1551       }
1552     }
1553   }
1554 
1555   /**
1556    * Record the location of the hbase:meta region as found in ZooKeeper.
1557    */
1558   private boolean recordMetaRegion() throws IOException {
1559     HRegionLocation metaLocation = connection.locateRegion(
1560       TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW);
1561 
1562     // Check if Meta region is valid and existing
1563     if (metaLocation == null || metaLocation.getRegionInfo() == null ||
1564         metaLocation.getHostname() == null) {
1565       errors.reportError(ERROR_CODE.NULL_META_REGION,
1566         "META region or some of its attributes are null.");
1567       return false;
1568     }
1569     ServerName sn;
1570     try {
1571       sn = getMetaRegionServerName();
1572     } catch (KeeperException e) {
1573       throw new IOException(e);
1574     }
1575     MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, System.currentTimeMillis());
1576     HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1577     if (hbckInfo == null) {
1578       regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1579     } else {
1580       hbckInfo.metaEntry = m;
1581     }
1582     return true;
1583   }
1584 
1585   private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1586     return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1587       @Override
1588       public void abort(String why, Throwable e) {
1589         LOG.error(why, e);
1590         System.exit(1);
1591       }
1592 
1593       @Override
1594       public boolean isAborted() {
1595         return false;
1596       }
1597 
1598     });
1599   }
1600 
1601   private ServerName getMetaRegionServerName()
1602   throws IOException, KeeperException {
1603     ZooKeeperWatcher zkw = createZooKeeperWatcher();
1604     ServerName sn = null;
1605     try {
1606       sn = new MetaTableLocator().getMetaRegionLocation(zkw);
1607     } finally {
1608       zkw.close();
1609     }
1610     return sn;
1611   }
1612 
1613   /**
1614    * Contacts each regionserver and fetches metadata about regions.
1615    * @param regionServerList - the list of region servers to connect to
1616    * @throws IOException if a remote or network exception occurs
1617    */
1618   void processRegionServers(Collection<ServerName> regionServerList)
1619     throws IOException, InterruptedException {
1620 
1621     List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1622     List<Future<Void>> workFutures;
1623 
1624     // loop to contact each region server in parallel
1625     for (ServerName rsinfo: regionServerList) {
1626       workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1627     }
1628 
1629     workFutures = executor.invokeAll(workItems);
1630 
1631     for(int i=0; i<workFutures.size(); i++) {
1632       WorkItemRegion item = workItems.get(i);
1633       Future<Void> f = workFutures.get(i);
1634       try {
1635         f.get();
1636       } catch(ExecutionException e) {
1637         LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1638             e.getCause());
1639       }
1640     }
1641   }
1642 
1643   /**
1644    * Check consistency of all regions that have been found in previous phases.
1645    */
1646   private void checkAndFixConsistency()
1647   throws IOException, KeeperException, InterruptedException {
1648     // Divide the checks in two phases. One for default/primary replicas and another
1649     // for the non-primary ones. Keeps code cleaner this way.
1650     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1651       if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1652         checkRegionConsistency(e.getKey(), e.getValue());
1653       }
1654     }
1655     boolean prevHdfsCheck = shouldCheckHdfs();
1656     setCheckHdfs(false); //replicas don't have any hdfs data
1657     // Run a pass over the replicas and fix any assignment issues that exist on the currently
1658     // deployed/undeployed replicas.
1659     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1660       if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1661         checkRegionConsistency(e.getKey(), e.getValue());
1662       }
1663     }
1664     setCheckHdfs(prevHdfsCheck);
1665   }
1666 
1667   private void preCheckPermission() throws IOException, AccessDeniedException {
1668     if (shouldIgnorePreCheckPermission()) {
1669       return;
1670     }
1671 
1672     Path hbaseDir = FSUtils.getRootDir(getConf());
1673     FileSystem fs = hbaseDir.getFileSystem(getConf());
1674     UserProvider userProvider = UserProvider.instantiate(getConf());
1675     UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1676     FileStatus[] files = fs.listStatus(hbaseDir);
1677     for (FileStatus file : files) {
1678       try {
1679         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1680       } catch (AccessDeniedException ace) {
1681         LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1682         errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1683           + " does not have write perms to " + file.getPath()
1684           + ". Please rerun hbck as hdfs user " + file.getOwner());
1685         throw ace;
1686       }
1687     }
1688   }
1689 
1690   /**
1691    * Deletes region from meta table
1692    */
1693   private void deleteMetaRegion(HbckInfo hi) throws IOException {
1694     deleteMetaRegion(hi.metaEntry.getRegionName());
1695   }
1696 
1697   /**
1698    * Deletes region from meta table
1699    */
1700   private void deleteMetaRegion(byte[] metaKey) throws IOException {
1701     Delete d = new Delete(metaKey);
1702     meta.delete(d);
1703     meta.flushCommits();
1704     LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1705   }
1706 
1707   /**
1708    * Reset the split parent region info in meta table
1709    */
1710   private void resetSplitParent(HbckInfo hi) throws IOException {
1711     RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
1712     Delete d = new Delete(hi.metaEntry.getRegionName());
1713     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1714     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1715     mutations.add(d);
1716 
1717     HRegionInfo hri = new HRegionInfo(hi.metaEntry);
1718     hri.setOffline(false);
1719     hri.setSplit(false);
1720     Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1721     mutations.add(p);
1722 
1723     meta.mutateRow(mutations);
1724     meta.flushCommits();
1725     LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
1726   }
1727 
1728   /**
1729    * This backwards-compatibility wrapper for permanently offlining a region
1730    * that should not be alive.  If the region server does not support the
1731    * "offline" method, it will use the closest unassign method instead.  This
1732    * will basically work until one attempts to disable or delete the affected
1733    * table.  The problem has to do with in-memory only master state, so
1734    * restarting the HMaster or failing over to another should fix this.
1735    */
1736   private void offline(byte[] regionName) throws IOException {
1737     String regionString = Bytes.toStringBinary(regionName);
1738     if (!rsSupportsOffline) {
1739       LOG.warn("Using unassign region " + regionString
1740           + " instead of using offline method, you should"
1741           + " restart HMaster after these repairs");
1742       admin.unassign(regionName, true);
1743       return;
1744     }
1745 
1746     // first time we assume the rs's supports #offline.
1747     try {
1748       LOG.info("Offlining region " + regionString);
1749       admin.offline(regionName);
1750     } catch (IOException ioe) {
1751       String notFoundMsg = "java.lang.NoSuchMethodException: " +
1752         "org.apache.hadoop.hbase.master.HMaster.offline([B)";
1753       if (ioe.getMessage().contains(notFoundMsg)) {
1754         LOG.warn("Using unassign region " + regionString
1755             + " instead of using offline method, you should"
1756             + " restart HMaster after these repairs");
1757         rsSupportsOffline = false; // in the future just use unassign
1758         admin.unassign(regionName, true);
1759         return;
1760       }
1761       throw ioe;
1762     }
1763   }
1764 
1765   private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
1766     undeployRegionsForHbi(hi);
1767     // undeploy replicas of the region (but only if the method is invoked for the primary)
1768     if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1769       return;
1770     }
1771     int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
1772     for (int i = 1; i < numReplicas; i++) {
1773       if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
1774       HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
1775           hi.getPrimaryHRIForDeployedReplica(), i);
1776       HbckInfo h = regionInfoMap.get(hri.getEncodedName());
1777       if (h != null) {
1778         undeployRegionsForHbi(h);
1779         //set skip checks; we undeployed it, and we don't want to evaluate this anymore
1780         //in consistency checks
1781         h.setSkipChecks(true);
1782       }
1783     }
1784   }
1785 
1786   private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
1787     for (OnlineEntry rse : hi.deployedEntries) {
1788       LOG.debug("Undeploy region "  + rse.hri + " from " + rse.hsa);
1789       try {
1790         HBaseFsckRepair.closeRegionSilentlyAndWait(connection, rse.hsa, rse.hri);
1791         offline(rse.hri.getRegionName());
1792       } catch (IOException ioe) {
1793         LOG.warn("Got exception when attempting to offline region "
1794             + Bytes.toString(rse.hri.getRegionName()), ioe);
1795       }
1796     }
1797   }
1798 
1799   /**
1800    * Attempts to undeploy a region from a region server based in information in
1801    * META.  Any operations that modify the file system should make sure that
1802    * its corresponding region is not deployed to prevent data races.
1803    *
1804    * A separate call is required to update the master in-memory region state
1805    * kept in the AssignementManager.  Because disable uses this state instead of
1806    * that found in META, we can't seem to cleanly disable/delete tables that
1807    * have been hbck fixed.  When used on a version of HBase that does not have
1808    * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master
1809    * restart or failover may be required.
1810    */
1811   private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
1812     if (hi.metaEntry == null && hi.hdfsEntry == null) {
1813       undeployRegions(hi);
1814       return;
1815     }
1816 
1817     // get assignment info and hregioninfo from meta.
1818     Get get = new Get(hi.getRegionName());
1819     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1820     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
1821     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
1822     // also get the locations of the replicas to close if the primary region is being closed
1823     if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1824       int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
1825       for (int i = 0; i < numReplicas; i++) {
1826         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
1827         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
1828       }
1829     }
1830     Result r = meta.get(get);
1831     RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
1832     if (rl == null) {
1833       LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
1834           " since meta does not have handle to reach it");
1835       return;
1836     }
1837     for (HRegionLocation h : rl.getRegionLocations()) {
1838       ServerName serverName = h.getServerName();
1839       if (serverName == null) {
1840         errors.reportError("Unable to close region "
1841             + hi.getRegionNameAsString() +  " because meta does not "
1842             + "have handle to reach it.");
1843         continue;
1844       }
1845       HRegionInfo hri = h.getRegionInfo();
1846       if (hri == null) {
1847         LOG.warn("Unable to close region " + hi.getRegionNameAsString()
1848             + " because hbase:meta had invalid or missing "
1849             + HConstants.CATALOG_FAMILY_STR + ":"
1850             + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
1851             + " qualifier value.");
1852         continue;
1853       }
1854       // close the region -- close files and remove assignment
1855       HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);
1856     }
1857   }
1858 
1859   private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
1860     KeeperException, InterruptedException {
1861     // If we are trying to fix the errors
1862     if (shouldFixAssignments()) {
1863       errors.print(msg);
1864       undeployRegions(hbi);
1865       setShouldRerun();
1866       HRegionInfo hri = hbi.getHdfsHRI();
1867       if (hri == null) {
1868         hri = hbi.metaEntry;
1869       }
1870       HBaseFsckRepair.fixUnassigned(admin, hri);
1871       HBaseFsckRepair.waitUntilAssigned(admin, hri);
1872 
1873       // also assign replicas if needed (do it only when this call operates on a primary replica)
1874       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
1875       int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
1876       for (int i = 1; i < replicationCount; i++) {
1877         hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
1878         HbckInfo h = regionInfoMap.get(hri.getEncodedName());
1879         if (h != null) {
1880           undeployRegions(h);
1881           //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore
1882           //in consistency checks
1883           h.setSkipChecks(true);
1884         }
1885         HBaseFsckRepair.fixUnassigned(admin, hri);
1886         HBaseFsckRepair.waitUntilAssigned(admin, hri);
1887       }
1888 
1889     }
1890   }
1891 
1892   /**
1893    * Check a single region for consistency and correct deployment.
1894    */
1895   private void checkRegionConsistency(final String key, final HbckInfo hbi)
1896   throws IOException, KeeperException, InterruptedException {
1897 
1898     if (hbi.isSkipChecks()) return;
1899     String descriptiveName = hbi.toString();
1900     boolean inMeta = hbi.metaEntry != null;
1901     // In case not checking HDFS, assume the region is on HDFS
1902     boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
1903     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
1904     boolean isDeployed = !hbi.deployedOn.isEmpty();
1905     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
1906     boolean deploymentMatchesMeta =
1907       hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
1908       hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
1909     boolean splitParent =
1910       (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
1911     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
1912     boolean recentlyModified = inHdfs &&
1913       hbi.getModTime() + timelag > System.currentTimeMillis();
1914 
1915     // ========== First the healthy cases =============
1916     if (hbi.containsOnlyHdfsEdits()) {
1917       return;
1918     }
1919     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
1920       return;
1921     } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
1922       LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
1923         "tabled that is not deployed");
1924       return;
1925     } else if (recentlyModified) {
1926       LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
1927       return;
1928     }
1929     // ========== Cases where the region is not in hbase:meta =============
1930     else if (!inMeta && !inHdfs && !isDeployed) {
1931       // We shouldn't have record of this region at all then!
1932       assert false : "Entry for region with no data";
1933     } else if (!inMeta && !inHdfs && isDeployed) {
1934       errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
1935           + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
1936           "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1937       if (shouldFixAssignments()) {
1938         undeployRegions(hbi);
1939       }
1940 
1941     } else if (!inMeta && inHdfs && !isDeployed) {
1942       if (hbi.isMerged()) {
1943         // This region has already been merged, the remaining hdfs file will be
1944         // cleaned by CatalogJanitor later
1945         hbi.setSkipChecks(true);
1946         LOG.info("Region " + descriptiveName
1947             + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
1948         return;
1949       }
1950       errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
1951           + descriptiveName + " on HDFS, but not listed in hbase:meta " +
1952           "or deployed on any region server");
1953       // restore region consistency of an adopted orphan
1954       if (shouldFixMeta()) {
1955         if (!hbi.isHdfsRegioninfoPresent()) {
1956           LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
1957               +  " in table integrity repair phase if -fixHdfsOrphans was" +
1958               " used.");
1959           return;
1960         }
1961 
1962         LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
1963         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
1964         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
1965             admin.getClusterStatus().getServers(), numReplicas);
1966 
1967         tryAssignmentRepair(hbi, "Trying to reassign region...");
1968       }
1969 
1970     } else if (!inMeta && inHdfs && isDeployed) {
1971       errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
1972           + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1973       debugLsr(hbi.getHdfsRegionDir());
1974       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1975         // for replicas, this means that we should undeploy the region (we would have
1976         // gone over the primaries and fixed meta holes in first phase under
1977         // checkAndFixConsistency; we shouldn't get the condition !inMeta at
1978         // this stage unless unwanted replica)
1979         if (shouldFixAssignments()) {
1980           undeployRegionsForHbi(hbi);
1981         }
1982       }
1983       if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1984         if (!hbi.isHdfsRegioninfoPresent()) {
1985           LOG.error("This should have been repaired in table integrity repair phase");
1986           return;
1987         }
1988 
1989         LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
1990         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
1991         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
1992             admin.getClusterStatus().getServers(), numReplicas);
1993         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
1994       }
1995 
1996     // ========== Cases where the region is in hbase:meta =============
1997     } else if (inMeta && inHdfs && !isDeployed && splitParent) {
1998       // check whether this is an actual error, or just transient state where parent
1999       // is not cleaned
2000       if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
2001         // check that split daughters are there
2002         HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
2003         HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
2004         if (infoA != null && infoB != null) {
2005           // we already processed or will process daughters. Move on, nothing to see here.
2006           hbi.setSkipChecks(true);
2007           return;
2008         }
2009       }
2010       errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
2011           + descriptiveName + " is a split parent in META, in HDFS, "
2012           + "and not deployed on any region server. This could be transient.");
2013       if (shouldFixSplitParents()) {
2014         setShouldRerun();
2015         resetSplitParent(hbi);
2016       }
2017     } else if (inMeta && !inHdfs && !isDeployed) {
2018       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
2019           + descriptiveName + " found in META, but not in HDFS "
2020           + "or deployed on any region server.");
2021       if (shouldFixMeta()) {
2022         deleteMetaRegion(hbi);
2023       }
2024     } else if (inMeta && !inHdfs && isDeployed) {
2025       errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
2026           + " found in META, but not in HDFS, " +
2027           "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2028       // We treat HDFS as ground truth.  Any information in meta is transient
2029       // and equivalent data can be regenerated.  So, lets unassign and remove
2030       // these problems from META.
2031       if (shouldFixAssignments()) {
2032         errors.print("Trying to fix unassigned region...");
2033         undeployRegions(hbi);
2034       }
2035       if (shouldFixMeta()) {
2036         // wait for it to complete
2037         deleteMetaRegion(hbi);
2038       }
2039     } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
2040       errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
2041           + " not deployed on any region server.");
2042       tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
2043     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
2044       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
2045           "Region " + descriptiveName + " should not be deployed according " +
2046           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
2047       if (shouldFixAssignments()) {
2048         errors.print("Trying to close the region " + descriptiveName);
2049         setShouldRerun();
2050         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2051       }
2052     } else if (inMeta && inHdfs && isMultiplyDeployed) {
2053       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
2054           + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
2055           + " but is multiply assigned to region servers " +
2056           Joiner.on(", ").join(hbi.deployedOn));
2057       // If we are trying to fix the errors
2058       if (shouldFixAssignments()) {
2059         errors.print("Trying to fix assignment error...");
2060         setShouldRerun();
2061         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2062       }
2063     } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
2064       errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
2065           + descriptiveName + " listed in hbase:meta on region server " +
2066           hbi.metaEntry.regionServer + " but found on region server " +
2067           hbi.deployedOn.get(0));
2068       // If we are trying to fix the errors
2069       if (shouldFixAssignments()) {
2070         errors.print("Trying to fix assignment error...");
2071         setShouldRerun();
2072         HBaseFsckRepair.fixMultiAssignment(connection, hbi.metaEntry, hbi.deployedOn);
2073         HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
2074       }
2075     } else {
2076       errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
2077           " is in an unforeseen state:" +
2078           " inMeta=" + inMeta +
2079           " inHdfs=" + inHdfs +
2080           " isDeployed=" + isDeployed +
2081           " isMultiplyDeployed=" + isMultiplyDeployed +
2082           " deploymentMatchesMeta=" + deploymentMatchesMeta +
2083           " shouldBeDeployed=" + shouldBeDeployed);
2084     }
2085   }
2086 
2087   /**
2088    * Checks tables integrity. Goes over all regions and scans the tables.
2089    * Collects all the pieces for each table and checks if there are missing,
2090    * repeated or overlapping ones.
2091    * @throws IOException
2092    */
2093   SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2094     tablesInfo = new TreeMap<TableName,TableInfo> ();
2095     LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2096     for (HbckInfo hbi : regionInfoMap.values()) {
2097       // Check only valid, working regions
2098       if (hbi.metaEntry == null) {
2099         // this assumes that consistency check has run loadMetaEntry
2100         Path p = hbi.getHdfsRegionDir();
2101         if (p == null) {
2102           errors.report("No regioninfo in Meta or HDFS. " + hbi);
2103         }
2104 
2105         // TODO test.
2106         continue;
2107       }
2108       if (hbi.metaEntry.regionServer == null) {
2109         errors.detail("Skipping region because no region server: " + hbi);
2110         continue;
2111       }
2112       if (hbi.metaEntry.isOffline()) {
2113         errors.detail("Skipping region because it is offline: " + hbi);
2114         continue;
2115       }
2116       if (hbi.containsOnlyHdfsEdits()) {
2117         errors.detail("Skipping region because it only contains edits" + hbi);
2118         continue;
2119       }
2120 
2121       // Missing regionDir or over-deployment is checked elsewhere. Include
2122       // these cases in modTInfo, so we can evaluate those regions as part of
2123       // the region chain in META
2124       //if (hbi.foundRegionDir == null) continue;
2125       //if (hbi.deployedOn.size() != 1) continue;
2126       if (hbi.deployedOn.size() == 0) continue;
2127 
2128       // We should be safe here
2129       TableName tableName = hbi.metaEntry.getTable();
2130       TableInfo modTInfo = tablesInfo.get(tableName);
2131       if (modTInfo == null) {
2132         modTInfo = new TableInfo(tableName);
2133       }
2134       for (ServerName server : hbi.deployedOn) {
2135         modTInfo.addServer(server);
2136       }
2137 
2138       if (!hbi.isSkipChecks()) {
2139         modTInfo.addRegionInfo(hbi);
2140       }
2141 
2142       tablesInfo.put(tableName, modTInfo);
2143     }
2144 
2145     loadTableInfosForTablesWithNoRegion();
2146 
2147     for (TableInfo tInfo : tablesInfo.values()) {
2148       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2149       if (!tInfo.checkRegionChain(handler)) {
2150         errors.report("Found inconsistency in table " + tInfo.getName());
2151       }
2152     }
2153     return tablesInfo;
2154   }
2155 
2156   /** Loads table info's for tables that may not have been included, since there are no
2157    * regions reported for the table, but table dir is there in hdfs
2158    */
2159   private void loadTableInfosForTablesWithNoRegion() throws IOException {
2160     Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2161     for (HTableDescriptor htd : allTables.values()) {
2162       if (checkMetaOnly && !htd.isMetaTable()) {
2163         continue;
2164       }
2165 
2166       TableName tableName = htd.getTableName();
2167       if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2168         TableInfo tableInfo = new TableInfo(tableName);
2169         tableInfo.htds.add(htd);
2170         tablesInfo.put(htd.getTableName(), tableInfo);
2171       }
2172     }
2173   }
2174 
2175   /**
2176    * Merge hdfs data by moving from contained HbckInfo into targetRegionDir.
2177    * @return number of file move fixes done to merge regions.
2178    */
2179   public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2180     int fileMoves = 0;
2181     String thread = Thread.currentThread().getName();
2182     LOG.debug("[" + thread + "] Contained region dir after close and pause");
2183     debugLsr(contained.getHdfsRegionDir());
2184 
2185     // rename the contained into the container.
2186     FileSystem fs = targetRegionDir.getFileSystem(getConf());
2187     FileStatus[] dirs = null;
2188     try {
2189       dirs = fs.listStatus(contained.getHdfsRegionDir());
2190     } catch (FileNotFoundException fnfe) {
2191       // region we are attempting to merge in is not present!  Since this is a merge, there is
2192       // no harm skipping this region if it does not exist.
2193       if (!fs.exists(contained.getHdfsRegionDir())) {
2194         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2195             + " is missing. Assuming already sidelined or moved.");
2196       } else {
2197         sidelineRegionDir(fs, contained);
2198       }
2199       return fileMoves;
2200     }
2201 
2202     if (dirs == null) {
2203       if (!fs.exists(contained.getHdfsRegionDir())) {
2204         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2205             + " already sidelined.");
2206       } else {
2207         sidelineRegionDir(fs, contained);
2208       }
2209       return fileMoves;
2210     }
2211 
2212     for (FileStatus cf : dirs) {
2213       Path src = cf.getPath();
2214       Path dst =  new Path(targetRegionDir, src.getName());
2215 
2216       if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2217         // do not copy the old .regioninfo file.
2218         continue;
2219       }
2220 
2221       if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2222         // do not copy the .oldlogs files
2223         continue;
2224       }
2225 
2226       LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2227       // FileSystem.rename is inconsistent with directories -- if the
2228       // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2229       // it moves the src into the dst dir resulting in (foo/a/b).  If
2230       // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2231       for (FileStatus hfile : fs.listStatus(src)) {
2232         boolean success = fs.rename(hfile.getPath(), dst);
2233         if (success) {
2234           fileMoves++;
2235         }
2236       }
2237       LOG.debug("[" + thread + "] Sideline directory contents:");
2238       debugLsr(targetRegionDir);
2239     }
2240 
2241     // if all success.
2242     sidelineRegionDir(fs, contained);
2243     LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2244         getSidelineDir());
2245     debugLsr(contained.getHdfsRegionDir());
2246 
2247     return fileMoves;
2248   }
2249 
2250 
2251   static class WorkItemOverlapMerge implements Callable<Void> {
2252     private TableIntegrityErrorHandler handler;
2253     Collection<HbckInfo> overlapgroup;
2254 
2255     WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2256       this.handler = handler;
2257       this.overlapgroup = overlapgroup;
2258     }
2259 
2260     @Override
2261     public Void call() throws Exception {
2262       handler.handleOverlapGroup(overlapgroup);
2263       return null;
2264     }
2265   };
2266 
2267 
2268   /**
2269    * Maintain information about a particular table.
2270    */
2271   public class TableInfo {
2272     TableName tableName;
2273     TreeSet <ServerName> deployedOn;
2274 
2275     // backwards regions
2276     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2277 
2278     // sidelined big overlapped regions
2279     final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2280 
2281     // region split calculator
2282     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2283 
2284     // Histogram of different HTableDescriptors found.  Ideally there is only one!
2285     final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2286 
2287     // key = start split, values = set of splits in problem group
2288     final Multimap<byte[], HbckInfo> overlapGroups =
2289       TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2290 
2291     TableInfo(TableName name) {
2292       this.tableName = name;
2293       deployedOn = new TreeSet <ServerName>();
2294     }
2295 
2296     /**
2297      * @return descriptor common to all regions.  null if are none or multiple!
2298      */
2299     private HTableDescriptor getHTD() {
2300       if (htds.size() == 1) {
2301         return (HTableDescriptor)htds.toArray()[0];
2302       } else {
2303         LOG.error("None/Multiple table descriptors found for table '"
2304           + tableName + "' regions: " + htds);
2305       }
2306       return null;
2307     }
2308 
2309     public void addRegionInfo(HbckInfo hir) {
2310       if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2311         // end key is absolute end key, just add it.
2312         // ignore replicas other than primary for these checks
2313         if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2314         return;
2315       }
2316 
2317       // if not the absolute end key, check for cycle
2318       if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2319         errors.reportError(
2320             ERROR_CODE.REGION_CYCLE,
2321             String.format("The endkey for this region comes before the "
2322                 + "startkey, startkey=%s, endkey=%s",
2323                 Bytes.toStringBinary(hir.getStartKey()),
2324                 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2325         backwards.add(hir);
2326         return;
2327       }
2328 
2329       // main case, add to split calculator
2330       // ignore replicas other than primary for these checks
2331       if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2332     }
2333 
2334     public void addServer(ServerName server) {
2335       this.deployedOn.add(server);
2336     }
2337 
2338     public TableName getName() {
2339       return tableName;
2340     }
2341 
2342     public int getNumRegions() {
2343       return sc.getStarts().size() + backwards.size();
2344     }
2345 
2346     private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2347       ErrorReporter errors;
2348 
2349       IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2350         this.errors = errors;
2351         setTableInfo(ti);
2352       }
2353 
2354       @Override
2355       public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2356         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2357             "First region should start with an empty key.  You need to "
2358             + " create a new region and regioninfo in HDFS to plug the hole.",
2359             getTableInfo(), hi);
2360       }
2361 
2362       @Override
2363       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2364         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2365             "Last region should end with an empty key. You need to "
2366                 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2367       }
2368 
2369       @Override
2370       public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2371         errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2372             "Region has the same start and end key.", getTableInfo(), hi);
2373       }
2374 
2375       @Override
2376       public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2377         byte[] key = r1.getStartKey();
2378         // dup start key
2379         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2380             "Multiple regions have the same startkey: "
2381             + Bytes.toStringBinary(key), getTableInfo(), r1);
2382         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2383             "Multiple regions have the same startkey: "
2384             + Bytes.toStringBinary(key), getTableInfo(), r2);
2385       }
2386 
2387       @Override
2388       public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2389         errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2390             "There is an overlap in the region chain.",
2391             getTableInfo(), hi1, hi2);
2392       }
2393 
2394       @Override
2395       public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2396         errors.reportError(
2397             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2398             "There is a hole in the region chain between "
2399                 + Bytes.toStringBinary(holeStart) + " and "
2400                 + Bytes.toStringBinary(holeStop)
2401                 + ".  You need to create a new .regioninfo and region "
2402                 + "dir in hdfs to plug the hole.");
2403       }
2404     };
2405 
2406     /**
2407      * This handler fixes integrity errors from hdfs information.  There are
2408      * basically three classes of integrity problems 1) holes, 2) overlaps, and
2409      * 3) invalid regions.
2410      *
2411      * This class overrides methods that fix holes and the overlap group case.
2412      * Individual cases of particular overlaps are handled by the general
2413      * overlap group merge repair case.
2414      *
2415      * If hbase is online, this forces regions offline before doing merge
2416      * operations.
2417      */
2418     private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2419       Configuration conf;
2420 
2421       boolean fixOverlaps = true;
2422 
2423       HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2424           boolean fixHoles, boolean fixOverlaps) {
2425         super(ti, errors);
2426         this.conf = conf;
2427         this.fixOverlaps = fixOverlaps;
2428         // TODO properly use fixHoles
2429       }
2430 
2431       /**
2432        * This is a special case hole -- when the first region of a table is
2433        * missing from META, HBase doesn't acknowledge the existance of the
2434        * table.
2435        */
2436       @Override
2437       public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2438         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2439             "First region should start with an empty key.  Creating a new " +
2440             "region and regioninfo in HDFS to plug the hole.",
2441             getTableInfo(), next);
2442         HTableDescriptor htd = getTableInfo().getHTD();
2443         // from special EMPTY_START_ROW to next region's startKey
2444         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2445             HConstants.EMPTY_START_ROW, next.getStartKey());
2446 
2447         // TODO test
2448         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2449         LOG.info("Table region start key was not empty.  Created new empty region: "
2450             + newRegion + " " +region);
2451         fixes++;
2452       }
2453 
2454       @Override
2455       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2456         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2457             "Last region should end with an empty key.  Creating a new "
2458                 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2459         HTableDescriptor htd = getTableInfo().getHTD();
2460         // from curEndKey to EMPTY_START_ROW
2461         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2462             HConstants.EMPTY_START_ROW);
2463 
2464         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2465         LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
2466             + " " + region);
2467         fixes++;
2468       }
2469 
2470       /**
2471        * There is a hole in the hdfs regions that violates the table integrity
2472        * rules.  Create a new empty region that patches the hole.
2473        */
2474       @Override
2475       public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2476         errors.reportError(
2477             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2478             "There is a hole in the region chain between "
2479                 + Bytes.toStringBinary(holeStartKey) + " and "
2480                 + Bytes.toStringBinary(holeStopKey)
2481                 + ".  Creating a new regioninfo and region "
2482                 + "dir in hdfs to plug the hole.");
2483         HTableDescriptor htd = getTableInfo().getHTD();
2484         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2485         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2486         LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2487         fixes++;
2488       }
2489 
2490       /**
2491        * This takes set of overlapping regions and merges them into a single
2492        * region.  This covers cases like degenerate regions, shared start key,
2493        * general overlaps, duplicate ranges, and partial overlapping regions.
2494        *
2495        * Cases:
2496        * - Clean regions that overlap
2497        * - Only .oldlogs regions (can't find start/stop range, or figure out)
2498        *
2499        * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
2500        */
2501       @Override
2502       public void handleOverlapGroup(Collection<HbckInfo> overlap)
2503           throws IOException {
2504         Preconditions.checkNotNull(overlap);
2505         Preconditions.checkArgument(overlap.size() >0);
2506 
2507         if (!this.fixOverlaps) {
2508           LOG.warn("Not attempting to repair overlaps.");
2509           return;
2510         }
2511 
2512         if (overlap.size() > maxMerge) {
2513           LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2514             "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2515           if (sidelineBigOverlaps) {
2516             // we only sideline big overlapped groups that exceeds the max number of regions to merge
2517             sidelineBigOverlaps(overlap);
2518           }
2519           return;
2520         }
2521 
2522         mergeOverlaps(overlap);
2523       }
2524 
2525       void mergeOverlaps(Collection<HbckInfo> overlap)
2526           throws IOException {
2527         String thread = Thread.currentThread().getName();
2528         LOG.info("== [" + thread + "] Merging regions into one region: "
2529           + Joiner.on(",").join(overlap));
2530         // get the min / max range and close all concerned regions
2531         Pair<byte[], byte[]> range = null;
2532         for (HbckInfo hi : overlap) {
2533           if (range == null) {
2534             range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2535           } else {
2536             if (RegionSplitCalculator.BYTES_COMPARATOR
2537                 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2538               range.setFirst(hi.getStartKey());
2539             }
2540             if (RegionSplitCalculator.BYTES_COMPARATOR
2541                 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2542               range.setSecond(hi.getEndKey());
2543             }
2544           }
2545           // need to close files so delete can happen.
2546           LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
2547           LOG.debug("[" + thread + "] Contained region dir before close");
2548           debugLsr(hi.getHdfsRegionDir());
2549           try {
2550             LOG.info("[" + thread + "] Closing region: " + hi);
2551             closeRegion(hi);
2552           } catch (IOException ioe) {
2553             LOG.warn("[" + thread + "] Was unable to close region " + hi
2554               + ".  Just continuing... ", ioe);
2555           } catch (InterruptedException e) {
2556             LOG.warn("[" + thread + "] Was unable to close region " + hi
2557               + ".  Just continuing... ", e);
2558           }
2559 
2560           try {
2561             LOG.info("[" + thread + "] Offlining region: " + hi);
2562             offline(hi.getRegionName());
2563           } catch (IOException ioe) {
2564             LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2565               + ".  Just continuing... ", ioe);
2566           }
2567         }
2568 
2569         // create new empty container region.
2570         HTableDescriptor htd = getTableInfo().getHTD();
2571         // from start key to end Key
2572         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
2573             range.getSecond());
2574         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2575         LOG.info("[" + thread + "] Created new empty container region: " +
2576             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
2577         debugLsr(region.getRegionFileSystem().getRegionDir());
2578 
2579         // all target regions are closed, should be able to safely cleanup.
2580         boolean didFix= false;
2581         Path target = region.getRegionFileSystem().getRegionDir();
2582         for (HbckInfo contained : overlap) {
2583           LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
2584           int merges = mergeRegionDirs(target, contained);
2585           if (merges > 0) {
2586             didFix = true;
2587           }
2588         }
2589         if (didFix) {
2590           fixes++;
2591         }
2592       }
2593 
2594       /**
2595        * Sideline some regions in a big overlap group so that it
2596        * will have fewer regions, and it is easier to merge them later on.
2597        *
2598        * @param bigOverlap the overlapped group with regions more than maxMerge
2599        * @throws IOException
2600        */
2601       void sidelineBigOverlaps(
2602           Collection<HbckInfo> bigOverlap) throws IOException {
2603         int overlapsToSideline = bigOverlap.size() - maxMerge;
2604         if (overlapsToSideline > maxOverlapsToSideline) {
2605           overlapsToSideline = maxOverlapsToSideline;
2606         }
2607         List<HbckInfo> regionsToSideline =
2608           RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
2609         FileSystem fs = FileSystem.get(conf);
2610         for (HbckInfo regionToSideline: regionsToSideline) {
2611           try {
2612             LOG.info("Closing region: " + regionToSideline);
2613             closeRegion(regionToSideline);
2614           } catch (IOException ioe) {
2615             LOG.warn("Was unable to close region " + regionToSideline
2616               + ".  Just continuing... ", ioe);
2617           } catch (InterruptedException e) {
2618             LOG.warn("Was unable to close region " + regionToSideline
2619               + ".  Just continuing... ", e);
2620           }
2621 
2622           try {
2623             LOG.info("Offlining region: " + regionToSideline);
2624             offline(regionToSideline.getRegionName());
2625           } catch (IOException ioe) {
2626             LOG.warn("Unable to offline region from master: " + regionToSideline
2627               + ".  Just continuing... ", ioe);
2628           }
2629 
2630           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
2631           Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
2632           if (sidelineRegionDir != null) {
2633             sidelinedRegions.put(sidelineRegionDir, regionToSideline);
2634             LOG.info("After sidelined big overlapped region: "
2635               + regionToSideline.getRegionNameAsString()
2636               + " to " + sidelineRegionDir.toString());
2637             fixes++;
2638           }
2639         }
2640       }
2641     }
2642 
2643     /**
2644      * Check the region chain (from META) of this table.  We are looking for
2645      * holes, overlaps, and cycles.
2646      * @return false if there are errors
2647      * @throws IOException
2648      */
2649     public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
2650       // When table is disabled no need to check for the region chain. Some of the regions
2651       // accidently if deployed, this below code might report some issues like missing start
2652       // or end regions or region hole in chain and may try to fix which is unwanted.
2653       if (disabledTables.contains(this.tableName)) {
2654         return true;
2655       }
2656       int originalErrorsCount = errors.getErrorList().size();
2657       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
2658       SortedSet<byte[]> splits = sc.getSplits();
2659 
2660       byte[] prevKey = null;
2661       byte[] problemKey = null;
2662 
2663       if (splits.size() == 0) {
2664         // no region for this table
2665         handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
2666       }
2667 
2668       for (byte[] key : splits) {
2669         Collection<HbckInfo> ranges = regions.get(key);
2670         if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
2671           for (HbckInfo rng : ranges) {
2672             handler.handleRegionStartKeyNotEmpty(rng);
2673           }
2674         }
2675 
2676         // check for degenerate ranges
2677         for (HbckInfo rng : ranges) {
2678           // special endkey case converts '' to null
2679           byte[] endKey = rng.getEndKey();
2680           endKey = (endKey.length == 0) ? null : endKey;
2681           if (Bytes.equals(rng.getStartKey(),endKey)) {
2682             handler.handleDegenerateRegion(rng);
2683           }
2684         }
2685 
2686         if (ranges.size() == 1) {
2687           // this split key is ok -- no overlap, not a hole.
2688           if (problemKey != null) {
2689             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2690           }
2691           problemKey = null; // fell through, no more problem.
2692         } else if (ranges.size() > 1) {
2693           // set the new problem key group name, if already have problem key, just
2694           // keep using it.
2695           if (problemKey == null) {
2696             // only for overlap regions.
2697             LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
2698             problemKey = key;
2699           }
2700           overlapGroups.putAll(problemKey, ranges);
2701 
2702           // record errors
2703           ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
2704           //  this dumb and n^2 but this shouldn't happen often
2705           for (HbckInfo r1 : ranges) {
2706             if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2707             subRange.remove(r1);
2708             for (HbckInfo r2 : subRange) {
2709               if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2710               if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
2711                 handler.handleDuplicateStartKeys(r1,r2);
2712               } else {
2713                 // overlap
2714                 handler.handleOverlapInRegionChain(r1, r2);
2715               }
2716             }
2717           }
2718 
2719         } else if (ranges.size() == 0) {
2720           if (problemKey != null) {
2721             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2722           }
2723           problemKey = null;
2724 
2725           byte[] holeStopKey = sc.getSplits().higher(key);
2726           // if higher key is null we reached the top.
2727           if (holeStopKey != null) {
2728             // hole
2729             handler.handleHoleInRegionChain(key, holeStopKey);
2730           }
2731         }
2732         prevKey = key;
2733       }
2734 
2735       // When the last region of a table is proper and having an empty end key, 'prevKey'
2736       // will be null.
2737       if (prevKey != null) {
2738         handler.handleRegionEndKeyNotEmpty(prevKey);
2739       }
2740 
2741       // TODO fold this into the TableIntegrityHandler
2742       if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
2743         LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
2744             " false to run serially.");
2745         boolean ok = handleOverlapsParallel(handler, prevKey);
2746         if (!ok) {
2747           return false;
2748         }
2749       } else {
2750         LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
2751             " true to run in parallel.");
2752         for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2753           handler.handleOverlapGroup(overlap);
2754         }
2755       }
2756 
2757       if (details) {
2758         // do full region split map dump
2759         errors.print("---- Table '"  +  this.tableName
2760             + "': region split map");
2761         dump(splits, regions);
2762         errors.print("---- Table '"  +  this.tableName
2763             + "': overlap groups");
2764         dumpOverlapProblems(overlapGroups);
2765         errors.print("There are " + overlapGroups.keySet().size()
2766             + " overlap groups with " + overlapGroups.size()
2767             + " overlapping regions");
2768       }
2769       if (!sidelinedRegions.isEmpty()) {
2770         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
2771         errors.print("---- Table '"  +  this.tableName
2772             + "': sidelined big overlapped regions");
2773         dumpSidelinedRegions(sidelinedRegions);
2774       }
2775       return errors.getErrorList().size() == originalErrorsCount;
2776     }
2777 
2778     private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
2779         throws IOException {
2780       // we parallelize overlap handler for the case we have lots of groups to fix.  We can
2781       // safely assume each group is independent.
2782       List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
2783       List<Future<Void>> rets;
2784       for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2785         //
2786         merges.add(new WorkItemOverlapMerge(overlap, handler));
2787       }
2788       try {
2789         rets = executor.invokeAll(merges);
2790       } catch (InterruptedException e) {
2791         LOG.error("Overlap merges were interrupted", e);
2792         return false;
2793       }
2794       for(int i=0; i<merges.size(); i++) {
2795         WorkItemOverlapMerge work = merges.get(i);
2796         Future<Void> f = rets.get(i);
2797         try {
2798           f.get();
2799         } catch(ExecutionException e) {
2800           LOG.warn("Failed to merge overlap group" + work, e.getCause());
2801         } catch (InterruptedException e) {
2802           LOG.error("Waiting for overlap merges was interrupted", e);
2803           return false;
2804         }
2805       }
2806       return true;
2807     }
2808 
2809     /**
2810      * This dumps data in a visually reasonable way for visual debugging
2811      *
2812      * @param splits
2813      * @param regions
2814      */
2815     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
2816       // we display this way because the last end key should be displayed as well.
2817       StringBuilder sb = new StringBuilder();
2818       for (byte[] k : splits) {
2819         sb.setLength(0); // clear out existing buffer, if any.
2820         sb.append(Bytes.toStringBinary(k) + ":\t");
2821         for (HbckInfo r : regions.get(k)) {
2822           sb.append("[ "+ r.toString() + ", "
2823               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
2824         }
2825         errors.print(sb.toString());
2826       }
2827     }
2828   }
2829 
2830   public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
2831     // we display this way because the last end key should be displayed as
2832     // well.
2833     for (byte[] k : regions.keySet()) {
2834       errors.print(Bytes.toStringBinary(k) + ":");
2835       for (HbckInfo r : regions.get(k)) {
2836         errors.print("[ " + r.toString() + ", "
2837             + Bytes.toStringBinary(r.getEndKey()) + "]");
2838       }
2839       errors.print("----");
2840     }
2841   }
2842 
2843   public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
2844     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
2845       TableName tableName = entry.getValue().getTableName();
2846       Path path = entry.getKey();
2847       errors.print("This sidelined region dir should be bulk loaded: "
2848         + path.toString());
2849       errors.print("Bulk load command looks like: "
2850         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
2851         + path.toUri().getPath() + " "+ tableName);
2852     }
2853   }
2854 
2855   public Multimap<byte[], HbckInfo> getOverlapGroups(
2856       TableName table) {
2857     TableInfo ti = tablesInfo.get(table);
2858     return ti.overlapGroups;
2859   }
2860 
2861   /**
2862    * Return a list of user-space table names whose metadata have not been
2863    * modified in the last few milliseconds specified by timelag
2864    * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
2865    * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
2866    * milliseconds specified by timelag, then the table is a candidate to be returned.
2867    * @return tables that have not been modified recently
2868    * @throws IOException if an error is encountered
2869    */
2870   HTableDescriptor[] getTables(AtomicInteger numSkipped) {
2871     List<TableName> tableNames = new ArrayList<TableName>();
2872     long now = System.currentTimeMillis();
2873 
2874     for (HbckInfo hbi : regionInfoMap.values()) {
2875       MetaEntry info = hbi.metaEntry;
2876 
2877       // if the start key is zero, then we have found the first region of a table.
2878       // pick only those tables that were not modified in the last few milliseconds.
2879       if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
2880         if (info.modTime + timelag < now) {
2881           tableNames.add(info.getTable());
2882         } else {
2883           numSkipped.incrementAndGet(); // one more in-flux table
2884         }
2885       }
2886     }
2887     return getHTableDescriptors(tableNames);
2888   }
2889 
2890   HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
2891     HTableDescriptor[] htd = new HTableDescriptor[0];
2892     Admin admin = null;
2893     try {
2894       LOG.info("getHTableDescriptors == tableNames => " + tableNames);
2895       admin = new HBaseAdmin(getConf());
2896       htd = admin.getTableDescriptorsByTableName(tableNames);
2897     } catch (IOException e) {
2898       LOG.debug("Exception getting table descriptors", e);
2899     } finally {
2900       if (admin != null) {
2901         try {
2902           admin.close();
2903         } catch (IOException e) {
2904           LOG.debug("Exception closing HBaseAdmin", e);
2905         }
2906       }
2907     }
2908     return htd;
2909   }
2910 
2911   /**
2912    * Gets the entry in regionInfo corresponding to the the given encoded
2913    * region name. If the region has not been seen yet, a new entry is added
2914    * and returned.
2915    */
2916   private synchronized HbckInfo getOrCreateInfo(String name) {
2917     HbckInfo hbi = regionInfoMap.get(name);
2918     if (hbi == null) {
2919       hbi = new HbckInfo(null);
2920       regionInfoMap.put(name, hbi);
2921     }
2922     return hbi;
2923   }
2924 
2925   private void checkAndFixTableLocks() throws IOException {
2926     TableLockChecker checker = new TableLockChecker(createZooKeeperWatcher(), errors);
2927     checker.checkTableLocks();
2928 
2929     if (this.fixTableLocks) {
2930       checker.fixExpiredTableLocks();
2931     }
2932   }
2933 
2934   /**
2935     * Check values in regionInfo for hbase:meta
2936     * Check if zero or more than one regions with hbase:meta are found.
2937     * If there are inconsistencies (i.e. zero or more than one regions
2938     * pretend to be holding the hbase:meta) try to fix that and report an error.
2939     * @throws IOException from HBaseFsckRepair functions
2940    * @throws KeeperException
2941    * @throws InterruptedException
2942     */
2943   boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
2944     List<HbckInfo> metaRegions = Lists.newArrayList();
2945     for (HbckInfo value : regionInfoMap.values()) {
2946       if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
2947         metaRegions.add(value);
2948       }
2949     }
2950 
2951     // There will be always one entry in regionInfoMap corresponding to hbase:meta
2952     // Check the deployed servers. It should be exactly one server.
2953     List<ServerName> servers = new ArrayList<ServerName>();
2954     HbckInfo metaHbckInfo = null;
2955     if (!metaRegions.isEmpty()) {
2956       metaHbckInfo = metaRegions.get(0);
2957       servers = metaHbckInfo.deployedOn;
2958     }
2959     if (servers.size() != 1) {
2960       if (servers.size() == 0) {
2961         errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta is not found on any region.");
2962         if (shouldFixAssignments()) {
2963           errors.print("Trying to fix a problem with hbase:meta..");
2964           setShouldRerun();
2965           // try to fix it (treat it as unassigned region)
2966           HBaseFsckRepair.fixUnassigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
2967           HBaseFsckRepair.waitUntilAssigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
2968         }
2969       } else if (servers.size() > 1) {
2970         errors
2971             .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta is found on more than one region.");
2972         if (shouldFixAssignments()) {
2973           if (metaHbckInfo == null) {
2974             errors.print(
2975               "Unable to fix problem with hbase:meta due to hbase:meta region info missing");
2976             return false;
2977           }
2978           errors.print("Trying to fix a problem with hbase:meta..");
2979           setShouldRerun();
2980           // try fix it (treat is a dupe assignment)
2981           HBaseFsckRepair.fixMultiAssignment(connection, metaHbckInfo.metaEntry, servers);
2982         }
2983       }
2984       // rerun hbck with hopefully fixed META
2985       return false;
2986     }
2987     // no errors, so continue normally
2988     return true;
2989   }
2990 
2991   /**
2992    * Scan hbase:meta, adding all regions found to the regionInfo map.
2993    * @throws IOException if an error is encountered
2994    */
2995   boolean loadMetaEntries() throws IOException {
2996     MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
2997       int countRecord = 1;
2998 
2999       // comparator to sort KeyValues with latest modtime
3000       final Comparator<Cell> comp = new Comparator<Cell>() {
3001         @Override
3002         public int compare(Cell k1, Cell k2) {
3003           return (int)(k1.getTimestamp() - k2.getTimestamp());
3004         }
3005       };
3006 
3007       @Override
3008       public boolean processRow(Result result) throws IOException {
3009         try {
3010 
3011           // record the latest modification of this META record
3012           long ts =  Collections.max(result.listCells(), comp).getTimestamp();
3013           RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
3014           if (rl == null) {
3015             emptyRegionInfoQualifiers.add(result);
3016             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3017               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3018             return true;
3019           }
3020           ServerName sn = null;
3021           if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
3022               rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
3023             emptyRegionInfoQualifiers.add(result);
3024             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
3025               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
3026             return true;
3027           }
3028           HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
3029           if (!(isTableIncluded(hri.getTable())
3030               || hri.isMetaRegion())) {
3031             return true;
3032           }
3033           PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
3034           for (HRegionLocation h : rl.getRegionLocations()) {
3035             if (h == null || h.getRegionInfo() == null) {
3036               continue;
3037             }
3038             sn = h.getServerName();
3039             hri = h.getRegionInfo();
3040 
3041             MetaEntry m = null;
3042             if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
3043               m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
3044             } else {
3045               m = new MetaEntry(hri, sn, ts, null, null);
3046             }
3047             HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
3048             if (previous == null) {
3049               regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
3050             } else if (previous.metaEntry == null) {
3051               previous.metaEntry = m;
3052             } else {
3053               throw new IOException("Two entries in hbase:meta are same " + previous);
3054             }
3055           }
3056           PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
3057           for (HRegionInfo mergeRegion : new HRegionInfo[] {
3058               mergeRegions.getFirst(), mergeRegions.getSecond() }) {
3059             if (mergeRegion != null) {
3060               // This region is already been merged
3061               HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
3062               hbInfo.setMerged(true);
3063             }
3064           }
3065 
3066           // show proof of progress to the user, once for every 100 records.
3067           if (countRecord % 100 == 0) {
3068             errors.progress();
3069           }
3070           countRecord++;
3071           return true;
3072         } catch (RuntimeException e) {
3073           LOG.error("Result=" + result);
3074           throw e;
3075         }
3076       }
3077     };
3078     if (!checkMetaOnly) {
3079       // Scan hbase:meta to pick up user regions
3080       MetaScanner.metaScan(getConf(), visitor);
3081     }
3082 
3083     errors.print("");
3084     return true;
3085   }
3086 
3087   /**
3088    * Stores the regioninfo entries scanned from META
3089    */
3090   static class MetaEntry extends HRegionInfo {
3091     ServerName regionServer;   // server hosting this region
3092     long modTime;          // timestamp of most recent modification metadata
3093     HRegionInfo splitA, splitB; //split daughters
3094 
3095     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3096       this(rinfo, regionServer, modTime, null, null);
3097     }
3098 
3099     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3100         HRegionInfo splitA, HRegionInfo splitB) {
3101       super(rinfo);
3102       this.regionServer = regionServer;
3103       this.modTime = modTime;
3104       this.splitA = splitA;
3105       this.splitB = splitB;
3106     }
3107 
3108     @Override
3109     public boolean equals(Object o) {
3110       boolean superEq = super.equals(o);
3111       if (!superEq) {
3112         return superEq;
3113       }
3114 
3115       MetaEntry me = (MetaEntry) o;
3116       if (!regionServer.equals(me.regionServer)) {
3117         return false;
3118       }
3119       return (modTime == me.modTime);
3120     }
3121 
3122     @Override
3123     public int hashCode() {
3124       int hash = Arrays.hashCode(getRegionName());
3125       hash ^= getRegionId();
3126       hash ^= Arrays.hashCode(getStartKey());
3127       hash ^= Arrays.hashCode(getEndKey());
3128       hash ^= Boolean.valueOf(isOffline()).hashCode();
3129       hash ^= getTable().hashCode();
3130       if (regionServer != null) {
3131         hash ^= regionServer.hashCode();
3132       }
3133       hash ^= modTime;
3134       return hash;
3135     }
3136   }
3137 
3138   /**
3139    * Stores the regioninfo entries from HDFS
3140    */
3141   static class HdfsEntry {
3142     HRegionInfo hri;
3143     Path hdfsRegionDir = null;
3144     long hdfsRegionDirModTime  = 0;
3145     boolean hdfsRegioninfoFilePresent = false;
3146     boolean hdfsOnlyEdits = false;
3147   }
3148 
3149   /**
3150    * Stores the regioninfo retrieved from Online region servers.
3151    */
3152   static class OnlineEntry {
3153     HRegionInfo hri;
3154     ServerName hsa;
3155 
3156     @Override
3157     public String toString() {
3158       return hsa.toString() + ";" + hri.getRegionNameAsString();
3159     }
3160   }
3161 
3162   /**
3163    * Maintain information about a particular region.  It gathers information
3164    * from three places -- HDFS, META, and region servers.
3165    */
3166   public static class HbckInfo implements KeyRange {
3167     private MetaEntry metaEntry = null; // info in META
3168     private HdfsEntry hdfsEntry = null; // info in HDFS
3169     private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
3170     private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
3171     private boolean skipChecks = false; // whether to skip further checks to this region info.
3172     private boolean isMerged = false;// whether this region has already been merged into another one
3173     private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3174     private HRegionInfo primaryHRIForDeployedReplica = null;
3175 
3176     HbckInfo(MetaEntry metaEntry) {
3177       this.metaEntry = metaEntry;
3178     }
3179 
3180     public int getReplicaId() {
3181       if (metaEntry != null) return metaEntry.getReplicaId();
3182       return deployedReplicaId;
3183     }
3184 
3185     public synchronized void addServer(HRegionInfo hri, ServerName server) {
3186       OnlineEntry rse = new OnlineEntry() ;
3187       rse.hri = hri;
3188       rse.hsa = server;
3189       this.deployedEntries.add(rse);
3190       this.deployedOn.add(server);
3191       // save the replicaId that we see deployed in the cluster
3192       this.deployedReplicaId = hri.getReplicaId();
3193       this.primaryHRIForDeployedReplica =
3194           RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3195     }
3196 
3197     @Override
3198     public synchronized String toString() {
3199       StringBuilder sb = new StringBuilder();
3200       sb.append("{ meta => ");
3201       sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3202       sb.append( ", hdfs => " + getHdfsRegionDir());
3203       sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3204       sb.append( ", replicaId => " + getReplicaId());
3205       sb.append(" }");
3206       return sb.toString();
3207     }
3208 
3209     @Override
3210     public byte[] getStartKey() {
3211       if (this.metaEntry != null) {
3212         return this.metaEntry.getStartKey();
3213       } else if (this.hdfsEntry != null) {
3214         return this.hdfsEntry.hri.getStartKey();
3215       } else {
3216         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3217         return null;
3218       }
3219     }
3220 
3221     @Override
3222     public byte[] getEndKey() {
3223       if (this.metaEntry != null) {
3224         return this.metaEntry.getEndKey();
3225       } else if (this.hdfsEntry != null) {
3226         return this.hdfsEntry.hri.getEndKey();
3227       } else {
3228         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3229         return null;
3230       }
3231     }
3232 
3233     public TableName getTableName() {
3234       if (this.metaEntry != null) {
3235         return this.metaEntry.getTable();
3236       } else if (this.hdfsEntry != null) {
3237         // we are only guaranteed to have a path and not an HRI for hdfsEntry,
3238         // so we get the name from the Path
3239         Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3240         return FSUtils.getTableName(tableDir);
3241       } else {
3242         // return the info from the first online/deployed hri
3243         for (OnlineEntry e : deployedEntries) {
3244           return e.hri.getTable();
3245         }
3246         return null;
3247       }
3248     }
3249 
3250     public String getRegionNameAsString() {
3251       if (metaEntry != null) {
3252         return metaEntry.getRegionNameAsString();
3253       } else if (hdfsEntry != null) {
3254         if (hdfsEntry.hri != null) {
3255           return hdfsEntry.hri.getRegionNameAsString();
3256         }
3257       } else {
3258         // return the info from the first online/deployed hri
3259         for (OnlineEntry e : deployedEntries) {
3260           return e.hri.getRegionNameAsString();
3261         }
3262       }
3263       return null;
3264     }
3265 
3266     public byte[] getRegionName() {
3267       if (metaEntry != null) {
3268         return metaEntry.getRegionName();
3269       } else if (hdfsEntry != null) {
3270         return hdfsEntry.hri.getRegionName();
3271       } else {
3272         // return the info from the first online/deployed hri
3273         for (OnlineEntry e : deployedEntries) {
3274           return e.hri.getRegionName();
3275         }
3276         return null;
3277       }
3278     }
3279 
3280     public HRegionInfo getPrimaryHRIForDeployedReplica() {
3281       return primaryHRIForDeployedReplica;
3282     }
3283 
3284     Path getHdfsRegionDir() {
3285       if (hdfsEntry == null) {
3286         return null;
3287       }
3288       return hdfsEntry.hdfsRegionDir;
3289     }
3290 
3291     boolean containsOnlyHdfsEdits() {
3292       if (hdfsEntry == null) {
3293         return false;
3294       }
3295       return hdfsEntry.hdfsOnlyEdits;
3296     }
3297 
3298     boolean isHdfsRegioninfoPresent() {
3299       if (hdfsEntry == null) {
3300         return false;
3301       }
3302       return hdfsEntry.hdfsRegioninfoFilePresent;
3303     }
3304 
3305     long getModTime() {
3306       if (hdfsEntry == null) {
3307         return 0;
3308       }
3309       return hdfsEntry.hdfsRegionDirModTime;
3310     }
3311 
3312     HRegionInfo getHdfsHRI() {
3313       if (hdfsEntry == null) {
3314         return null;
3315       }
3316       return hdfsEntry.hri;
3317     }
3318 
3319     public void setSkipChecks(boolean skipChecks) {
3320       this.skipChecks = skipChecks;
3321     }
3322 
3323     public boolean isSkipChecks() {
3324       return skipChecks;
3325     }
3326 
3327     public void setMerged(boolean isMerged) {
3328       this.isMerged = isMerged;
3329     }
3330 
3331     public boolean isMerged() {
3332       return this.isMerged;
3333     }
3334   }
3335 
3336   final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3337     @Override
3338     public int compare(HbckInfo l, HbckInfo r) {
3339       if (l == r) {
3340         // same instance
3341         return 0;
3342       }
3343 
3344       int tableCompare = l.getTableName().compareTo(r.getTableName());
3345       if (tableCompare != 0) {
3346         return tableCompare;
3347       }
3348 
3349       int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3350           l.getStartKey(), r.getStartKey());
3351       if (startComparison != 0) {
3352         return startComparison;
3353       }
3354 
3355       // Special case for absolute endkey
3356       byte[] endKey = r.getEndKey();
3357       endKey = (endKey.length == 0) ? null : endKey;
3358       byte[] endKey2 = l.getEndKey();
3359       endKey2 = (endKey2.length == 0) ? null : endKey2;
3360       int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3361           endKey2,  endKey);
3362 
3363       if (endComparison != 0) {
3364         return endComparison;
3365       }
3366 
3367       // use regionId as tiebreaker.
3368       // Null is considered after all possible values so make it bigger.
3369       if (l.hdfsEntry == null && r.hdfsEntry == null) {
3370         return 0;
3371       }
3372       if (l.hdfsEntry == null && r.hdfsEntry != null) {
3373         return 1;
3374       }
3375       // l.hdfsEntry must not be null
3376       if (r.hdfsEntry == null) {
3377         return -1;
3378       }
3379       // both l.hdfsEntry and r.hdfsEntry must not be null.
3380       return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3381     }
3382   };
3383 
3384   /**
3385    * Prints summary of all tables found on the system.
3386    */
3387   private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3388     StringBuilder sb = new StringBuilder();
3389     errors.print("Summary:");
3390     for (TableInfo tInfo : tablesInfo.values()) {
3391       if (errors.tableHasErrors(tInfo)) {
3392         errors.print("Table " + tInfo.getName() + " is inconsistent.");
3393       } else {
3394         errors.print("  " + tInfo.getName() + " is okay.");
3395       }
3396       errors.print("    Number of regions: " + tInfo.getNumRegions());
3397       sb.setLength(0); // clear out existing buffer, if any.
3398       sb.append("    Deployed on: ");
3399       for (ServerName server : tInfo.deployedOn) {
3400         sb.append(" " + server.toString());
3401       }
3402       errors.print(sb.toString());
3403     }
3404   }
3405 
3406   static ErrorReporter getErrorReporter(
3407       final Configuration conf) throws ClassNotFoundException {
3408     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3409     return ReflectionUtils.newInstance(reporter, conf);
3410   }
3411 
3412   public interface ErrorReporter {
3413     enum ERROR_CODE {
3414       UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3415       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
3416       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3417       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3418       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3419       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3420       WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR
3421     }
3422     void clear();
3423     void report(String message);
3424     void reportError(String message);
3425     void reportError(ERROR_CODE errorCode, String message);
3426     void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3427     void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3428     void reportError(
3429       ERROR_CODE errorCode,
3430       String message,
3431       TableInfo table,
3432       HbckInfo info1,
3433       HbckInfo info2
3434     );
3435     int summarize();
3436     void detail(String details);
3437     ArrayList<ERROR_CODE> getErrorList();
3438     void progress();
3439     void print(String message);
3440     void resetErrors();
3441     boolean tableHasErrors(TableInfo table);
3442   }
3443 
3444   static class PrintingErrorReporter implements ErrorReporter {
3445     public int errorCount = 0;
3446     private int showProgress;
3447 
3448     Set<TableInfo> errorTables = new HashSet<TableInfo>();
3449 
3450     // for use by unit tests to verify which errors were discovered
3451     private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3452 
3453     @Override
3454     public void clear() {
3455       errorTables.clear();
3456       errorList.clear();
3457       errorCount = 0;
3458     }
3459 
3460     @Override
3461     public synchronized void reportError(ERROR_CODE errorCode, String message) {
3462       if (errorCode == ERROR_CODE.WRONG_USAGE) {
3463         System.err.println(message);
3464         return;
3465       }
3466 
3467       errorList.add(errorCode);
3468       if (!summary) {
3469         System.out.println("ERROR: " + message);
3470       }
3471       errorCount++;
3472       showProgress = 0;
3473     }
3474 
3475     @Override
3476     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
3477       errorTables.add(table);
3478       reportError(errorCode, message);
3479     }
3480 
3481     @Override
3482     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3483                                          HbckInfo info) {
3484       errorTables.add(table);
3485       String reference = "(region " + info.getRegionNameAsString() + ")";
3486       reportError(errorCode, reference + " " + message);
3487     }
3488 
3489     @Override
3490     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3491                                          HbckInfo info1, HbckInfo info2) {
3492       errorTables.add(table);
3493       String reference = "(regions " + info1.getRegionNameAsString()
3494           + " and " + info2.getRegionNameAsString() + ")";
3495       reportError(errorCode, reference + " " + message);
3496     }
3497 
3498     @Override
3499     public synchronized void reportError(String message) {
3500       reportError(ERROR_CODE.UNKNOWN, message);
3501     }
3502 
3503     /**
3504      * Report error information, but do not increment the error count.  Intended for cases
3505      * where the actual error would have been reported previously.
3506      * @param message
3507      */
3508     @Override
3509     public synchronized void report(String message) {
3510       if (! summary) {
3511         System.out.println("ERROR: " + message);
3512       }
3513       showProgress = 0;
3514     }
3515 
3516     @Override
3517     public synchronized int summarize() {
3518       System.out.println(Integer.toString(errorCount) +
3519                          " inconsistencies detected.");
3520       if (errorCount == 0) {
3521         System.out.println("Status: OK");
3522         return 0;
3523       } else {
3524         System.out.println("Status: INCONSISTENT");
3525         return -1;
3526       }
3527     }
3528 
3529     @Override
3530     public ArrayList<ERROR_CODE> getErrorList() {
3531       return errorList;
3532     }
3533 
3534     @Override
3535     public synchronized void print(String message) {
3536       if (!summary) {
3537         System.out.println(message);
3538       }
3539     }
3540 
3541     @Override
3542     public boolean tableHasErrors(TableInfo table) {
3543       return errorTables.contains(table);
3544     }
3545 
3546     @Override
3547     public void resetErrors() {
3548       errorCount = 0;
3549     }
3550 
3551     @Override
3552     public synchronized void detail(String message) {
3553       if (details) {
3554         System.out.println(message);
3555       }
3556       showProgress = 0;
3557     }
3558 
3559     @Override
3560     public synchronized void progress() {
3561       if (showProgress++ == 10) {
3562         if (!summary) {
3563           System.out.print(".");
3564         }
3565         showProgress = 0;
3566       }
3567     }
3568   }
3569 
3570   /**
3571    * Contact a region server and get all information from it
3572    */
3573   static class WorkItemRegion implements Callable<Void> {
3574     private HBaseFsck hbck;
3575     private ServerName rsinfo;
3576     private ErrorReporter errors;
3577     private HConnection connection;
3578 
3579     WorkItemRegion(HBaseFsck hbck, ServerName info,
3580                    ErrorReporter errors, HConnection connection) {
3581       this.hbck = hbck;
3582       this.rsinfo = info;
3583       this.errors = errors;
3584       this.connection = connection;
3585     }
3586 
3587     @Override
3588     public synchronized Void call() throws IOException {
3589       errors.progress();
3590       try {
3591         BlockingInterface server = connection.getAdmin(rsinfo);
3592 
3593         // list all online regions from this region server
3594         List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
3595         regions = filterRegions(regions);
3596 
3597         if (details) {
3598           errors.detail("RegionServer: " + rsinfo.getServerName() +
3599                            " number of regions: " + regions.size());
3600           for (HRegionInfo rinfo: regions) {
3601             errors.detail("  " + rinfo.getRegionNameAsString() +
3602                              " id: " + rinfo.getRegionId() +
3603                              " encoded_name: " + rinfo.getEncodedName() +
3604                              " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
3605                              " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
3606           }
3607         }
3608 
3609         // check to see if the existence of this region matches the region in META
3610         for (HRegionInfo r:regions) {
3611           HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
3612           hbi.addServer(r, rsinfo);
3613         }
3614       } catch (IOException e) {          // unable to connect to the region server.
3615         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
3616           " Unable to fetch region information. " + e);
3617         throw e;
3618       }
3619       return null;
3620     }
3621 
3622     private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
3623       List<HRegionInfo> ret = Lists.newArrayList();
3624       for (HRegionInfo hri : regions) {
3625         if (hri.isMetaTable() || (!hbck.checkMetaOnly
3626             && hbck.isTableIncluded(hri.getTable()))) {
3627           ret.add(hri);
3628         }
3629       }
3630       return ret;
3631     }
3632   }
3633 
3634   /**
3635    * Contact hdfs and get all information about specified table directory into
3636    * regioninfo list.
3637    */
3638   static class WorkItemHdfsDir implements Callable<Void> {
3639     private HBaseFsck hbck;
3640     private FileStatus tableDir;
3641     private ErrorReporter errors;
3642     private FileSystem fs;
3643 
3644     WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
3645                     FileStatus status) {
3646       this.hbck = hbck;
3647       this.fs = fs;
3648       this.tableDir = status;
3649       this.errors = errors;
3650     }
3651 
3652     @Override
3653     public synchronized Void call() throws IOException {
3654       try {
3655         // level 2: <HBASE_DIR>/<table>/*
3656         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
3657         for (FileStatus regionDir : regionDirs) {
3658           String encodedName = regionDir.getPath().getName();
3659           // ignore directories that aren't hexadecimal
3660           if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
3661             continue;
3662           }
3663 
3664           LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
3665           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
3666           HdfsEntry he = new HdfsEntry();
3667           synchronized (hbi) {
3668             if (hbi.getHdfsRegionDir() != null) {
3669               errors.print("Directory " + encodedName + " duplicate??" +
3670                            hbi.getHdfsRegionDir());
3671             }
3672 
3673             he.hdfsRegionDir = regionDir.getPath();
3674             he.hdfsRegionDirModTime = regionDir.getModificationTime();
3675             Path regioninfoFile = new Path(he.hdfsRegionDir, HRegionFileSystem.REGION_INFO_FILE);
3676             he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
3677             // we add to orphan list when we attempt to read .regioninfo
3678 
3679             // Set a flag if this region contains only edits
3680             // This is special case if a region is left after split
3681             he.hdfsOnlyEdits = true;
3682             FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
3683             Path ePath = WALSplitter.getRegionDirRecoveredEditsDir(regionDir.getPath());
3684             for (FileStatus subDir : subDirs) {
3685               String sdName = subDir.getPath().getName();
3686               if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
3687                 he.hdfsOnlyEdits = false;
3688                 break;
3689               }
3690             }
3691             hbi.hdfsEntry = he;
3692           }
3693         }
3694       } catch (IOException e) {
3695         // unable to connect to the region server.
3696         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
3697             + tableDir.getPath().getName()
3698             + " Unable to fetch region information. " + e);
3699         throw e;
3700       }
3701       return null;
3702     }
3703   }
3704 
3705   /**
3706    * Contact hdfs and get all information about specified table directory into
3707    * regioninfo list.
3708    */
3709   static class WorkItemHdfsRegionInfo implements Callable<Void> {
3710     private HbckInfo hbi;
3711     private HBaseFsck hbck;
3712     private ErrorReporter errors;
3713 
3714     WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
3715       this.hbi = hbi;
3716       this.hbck = hbck;
3717       this.errors = errors;
3718     }
3719 
3720     @Override
3721     public synchronized Void call() throws IOException {
3722       // only load entries that haven't been loaded yet.
3723       if (hbi.getHdfsHRI() == null) {
3724         try {
3725           hbck.loadHdfsRegioninfo(hbi);
3726         } catch (IOException ioe) {
3727           String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
3728               + hbi.getTableName() + " in hdfs dir "
3729               + hbi.getHdfsRegionDir()
3730               + "!  It may be an invalid format or version file.  Treating as "
3731               + "an orphaned regiondir.";
3732           errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
3733           try {
3734             hbck.debugLsr(hbi.getHdfsRegionDir());
3735           } catch (IOException ioe2) {
3736             LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
3737             throw ioe2;
3738           }
3739           hbck.orphanHdfsDirs.add(hbi);
3740           throw ioe;
3741         }
3742       }
3743       return null;
3744     }
3745   };
3746 
3747   /**
3748    * Display the full report from fsck. This displays all live and dead region
3749    * servers, and all known regions.
3750    */
3751   public void setDisplayFullReport() {
3752     details = true;
3753   }
3754 
3755   /**
3756    * Set summary mode.
3757    * Print only summary of the tables and status (OK or INCONSISTENT)
3758    */
3759   void setSummary() {
3760     summary = true;
3761   }
3762 
3763   /**
3764    * Set hbase:meta check mode.
3765    * Print only info about hbase:meta table deployment/state
3766    */
3767   void setCheckMetaOnly() {
3768     checkMetaOnly = true;
3769   }
3770 
3771   /**
3772    * Set region boundaries check mode.
3773    */
3774   void setRegionBoundariesCheck() {
3775     checkRegionBoundaries = true;
3776   }
3777 
3778   /**
3779    * Set table locks fix mode.
3780    * Delete table locks held for a long time
3781    */
3782   public void setFixTableLocks(boolean shouldFix) {
3783     fixTableLocks = shouldFix;
3784     fixAny |= shouldFix;
3785   }
3786 
3787   /**
3788    * Check if we should rerun fsck again. This checks if we've tried to
3789    * fix something and we should rerun fsck tool again.
3790    * Display the full report from fsck. This displays all live and dead
3791    * region servers, and all known regions.
3792    */
3793   void setShouldRerun() {
3794     rerun = true;
3795   }
3796 
3797   boolean shouldRerun() {
3798     return rerun;
3799   }
3800 
3801   /**
3802    * Fix inconsistencies found by fsck. This should try to fix errors (if any)
3803    * found by fsck utility.
3804    */
3805   public void setFixAssignments(boolean shouldFix) {
3806     fixAssignments = shouldFix;
3807     fixAny |= shouldFix;
3808   }
3809 
3810   boolean shouldFixAssignments() {
3811     return fixAssignments;
3812   }
3813 
3814   public void setFixMeta(boolean shouldFix) {
3815     fixMeta = shouldFix;
3816     fixAny |= shouldFix;
3817   }
3818 
3819   boolean shouldFixMeta() {
3820     return fixMeta;
3821   }
3822 
3823   public void setFixEmptyMetaCells(boolean shouldFix) {
3824     fixEmptyMetaCells = shouldFix;
3825     fixAny |= shouldFix;
3826   }
3827 
3828   boolean shouldFixEmptyMetaCells() {
3829     return fixEmptyMetaCells;
3830   }
3831 
3832   public void setCheckHdfs(boolean checking) {
3833     checkHdfs = checking;
3834   }
3835 
3836   boolean shouldCheckHdfs() {
3837     return checkHdfs;
3838   }
3839 
3840   public void setFixHdfsHoles(boolean shouldFix) {
3841     fixHdfsHoles = shouldFix;
3842     fixAny |= shouldFix;
3843   }
3844 
3845   boolean shouldFixHdfsHoles() {
3846     return fixHdfsHoles;
3847   }
3848 
3849   public void setFixTableOrphans(boolean shouldFix) {
3850     fixTableOrphans = shouldFix;
3851     fixAny |= shouldFix;
3852   }
3853 
3854   boolean shouldFixTableOrphans() {
3855     return fixTableOrphans;
3856   }
3857 
3858   public void setFixHdfsOverlaps(boolean shouldFix) {
3859     fixHdfsOverlaps = shouldFix;
3860     fixAny |= shouldFix;
3861   }
3862 
3863   boolean shouldFixHdfsOverlaps() {
3864     return fixHdfsOverlaps;
3865   }
3866 
3867   public void setFixHdfsOrphans(boolean shouldFix) {
3868     fixHdfsOrphans = shouldFix;
3869     fixAny |= shouldFix;
3870   }
3871 
3872   boolean shouldFixHdfsOrphans() {
3873     return fixHdfsOrphans;
3874   }
3875 
3876   public void setFixVersionFile(boolean shouldFix) {
3877     fixVersionFile = shouldFix;
3878     fixAny |= shouldFix;
3879   }
3880 
3881   public boolean shouldFixVersionFile() {
3882     return fixVersionFile;
3883   }
3884 
3885   public void setSidelineBigOverlaps(boolean sbo) {
3886     this.sidelineBigOverlaps = sbo;
3887   }
3888 
3889   public boolean shouldSidelineBigOverlaps() {
3890     return sidelineBigOverlaps;
3891   }
3892 
3893   public void setFixSplitParents(boolean shouldFix) {
3894     fixSplitParents = shouldFix;
3895     fixAny |= shouldFix;
3896   }
3897 
3898   boolean shouldFixSplitParents() {
3899     return fixSplitParents;
3900   }
3901 
3902   public void setFixReferenceFiles(boolean shouldFix) {
3903     fixReferenceFiles = shouldFix;
3904     fixAny |= shouldFix;
3905   }
3906 
3907   boolean shouldFixReferenceFiles() {
3908     return fixReferenceFiles;
3909   }
3910 
3911   public boolean shouldIgnorePreCheckPermission() {
3912     return !fixAny || ignorePreCheckPermission;
3913   }
3914 
3915   public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
3916     this.ignorePreCheckPermission = ignorePreCheckPermission;
3917   }
3918 
3919   /**
3920    * @param mm maximum number of regions to merge into a single region.
3921    */
3922   public void setMaxMerge(int mm) {
3923     this.maxMerge = mm;
3924   }
3925 
3926   public int getMaxMerge() {
3927     return maxMerge;
3928   }
3929 
3930   public void setMaxOverlapsToSideline(int mo) {
3931     this.maxOverlapsToSideline = mo;
3932   }
3933 
3934   public int getMaxOverlapsToSideline() {
3935     return maxOverlapsToSideline;
3936   }
3937 
3938   /**
3939    * Only check/fix tables specified by the list,
3940    * Empty list means all tables are included.
3941    */
3942   boolean isTableIncluded(TableName table) {
3943     return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
3944   }
3945 
3946   public void includeTable(TableName table) {
3947     tablesIncluded.add(table);
3948   }
3949 
3950   Set<TableName> getIncludedTables() {
3951     return new HashSet<TableName>(tablesIncluded);
3952   }
3953 
3954   /**
3955    * We are interested in only those tables that have not changed their state in
3956    * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
3957    * @param seconds - the time in seconds
3958    */
3959   public void setTimeLag(long seconds) {
3960     timelag = seconds * 1000; // convert to milliseconds
3961   }
3962 
3963   /**
3964    *
3965    * @param sidelineDir - HDFS path to sideline data
3966    */
3967   public void setSidelineDir(String sidelineDir) {
3968     this.sidelineDir = new Path(sidelineDir);
3969   }
3970 
3971   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
3972     return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
3973   }
3974 
3975   public HFileCorruptionChecker getHFilecorruptionChecker() {
3976     return hfcc;
3977   }
3978 
3979   public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
3980     this.hfcc = hfcc;
3981   }
3982 
3983   public void setRetCode(int code) {
3984     this.retcode = code;
3985   }
3986 
3987   public int getRetCode() {
3988     return retcode;
3989   }
3990 
3991   protected HBaseFsck printUsageAndExit() {
3992     StringWriter sw = new StringWriter(2048);
3993     PrintWriter out = new PrintWriter(sw);
3994     out.println("Usage: fsck [opts] {only tables}");
3995     out.println(" where [opts] are:");
3996     out.println("   -help Display help options (this)");
3997     out.println("   -details Display full report of all regions.");
3998     out.println("   -timelag <timeInSeconds>  Process only regions that " +
3999                        " have not experienced any metadata updates in the last " +
4000                        " <timeInSeconds> seconds.");
4001     out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
4002         " before checking if the fix worked if run with -fix");
4003     out.println("   -summary Print only summary of the tables and status.");
4004     out.println("   -metaonly Only check the state of the hbase:meta table.");
4005     out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
4006     out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
4007 
4008     out.println("");
4009     out.println("  Metadata Repair options: (expert features, use with caution!)");
4010     out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
4011     out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
4012     out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
4013     out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
4014         + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
4015     out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
4016     out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
4017     out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
4018     out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
4019     out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
4020     out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
4021     out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
4022     out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
4023     out.println("   -fixSplitParents  Try to force offline split parents to be online.");
4024     out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
4025     out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
4026     out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
4027         + " (empty REGIONINFO_QUALIFIER rows)");
4028 
4029     out.println("");
4030     out.println("  Datafile Repair options: (expert features, use with caution!)");
4031     out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
4032     out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
4033 
4034     out.println("");
4035     out.println("  Metadata Repair shortcuts");
4036     out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
4037         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles -fixTableLocks");
4038     out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
4039 
4040     out.println("");
4041     out.println("  Table lock options");
4042     out.println("   -fixTableLocks    Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
4043 
4044     out.flush();
4045     errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
4046 
4047     setRetCode(-2);
4048     return this;
4049   }
4050 
4051   /**
4052    * Main program
4053    *
4054    * @param args
4055    * @throws Exception
4056    */
4057   public static void main(String[] args) throws Exception {
4058     // create a fsck object
4059     Configuration conf = HBaseConfiguration.create();
4060     Path hbasedir = FSUtils.getRootDir(conf);
4061     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
4062     FSUtils.setFsDefault(conf, new Path(defaultFs));
4063     int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
4064     System.exit(ret);
4065   }
4066 
4067   /**
4068    * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
4069    */
4070   static class HBaseFsckTool extends Configured implements Tool {
4071     HBaseFsckTool(Configuration conf) { super(conf); }
4072     @Override
4073     public int run(String[] args) throws Exception {
4074       HBaseFsck hbck = new HBaseFsck(getConf());
4075       hbck.exec(hbck.executor, args);
4076       return hbck.getRetCode();
4077     }
4078   };
4079 
4080 
4081   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
4082     ServiceException, InterruptedException {
4083     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
4084 
4085     boolean checkCorruptHFiles = false;
4086     boolean sidelineCorruptHFiles = false;
4087 
4088     // Process command-line args.
4089     for (int i = 0; i < args.length; i++) {
4090       String cmd = args[i];
4091       if (cmd.equals("-help") || cmd.equals("-h")) {
4092         return printUsageAndExit();
4093       } else if (cmd.equals("-details")) {
4094         setDisplayFullReport();
4095       } else if (cmd.equals("-timelag")) {
4096         if (i == args.length - 1) {
4097           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4098           return printUsageAndExit();
4099         }
4100         try {
4101           long timelag = Long.parseLong(args[i+1]);
4102           setTimeLag(timelag);
4103         } catch (NumberFormatException e) {
4104           errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4105           return printUsageAndExit();
4106         }
4107         i++;
4108       } else if (cmd.equals("-sleepBeforeRerun")) {
4109         if (i == args.length - 1) {
4110           errors.reportError(ERROR_CODE.WRONG_USAGE,
4111             "HBaseFsck: -sleepBeforeRerun needs a value.");
4112           return printUsageAndExit();
4113         }
4114         try {
4115           sleepBeforeRerun = Long.parseLong(args[i+1]);
4116         } catch (NumberFormatException e) {
4117           errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4118           return printUsageAndExit();
4119         }
4120         i++;
4121       } else if (cmd.equals("-sidelineDir")) {
4122         if (i == args.length - 1) {
4123           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4124           return printUsageAndExit();
4125         }
4126         i++;
4127         setSidelineDir(args[i]);
4128       } else if (cmd.equals("-fix")) {
4129         errors.reportError(ERROR_CODE.WRONG_USAGE,
4130           "This option is deprecated, please use  -fixAssignments instead.");
4131         setFixAssignments(true);
4132       } else if (cmd.equals("-fixAssignments")) {
4133         setFixAssignments(true);
4134       } else if (cmd.equals("-fixMeta")) {
4135         setFixMeta(true);
4136       } else if (cmd.equals("-noHdfsChecking")) {
4137         setCheckHdfs(false);
4138       } else if (cmd.equals("-fixHdfsHoles")) {
4139         setFixHdfsHoles(true);
4140       } else if (cmd.equals("-fixHdfsOrphans")) {
4141         setFixHdfsOrphans(true);
4142       } else if (cmd.equals("-fixTableOrphans")) {
4143         setFixTableOrphans(true);
4144       } else if (cmd.equals("-fixHdfsOverlaps")) {
4145         setFixHdfsOverlaps(true);
4146       } else if (cmd.equals("-fixVersionFile")) {
4147         setFixVersionFile(true);
4148       } else if (cmd.equals("-sidelineBigOverlaps")) {
4149         setSidelineBigOverlaps(true);
4150       } else if (cmd.equals("-fixSplitParents")) {
4151         setFixSplitParents(true);
4152       } else if (cmd.equals("-ignorePreCheckPermission")) {
4153         setIgnorePreCheckPermission(true);
4154       } else if (cmd.equals("-checkCorruptHFiles")) {
4155         checkCorruptHFiles = true;
4156       } else if (cmd.equals("-sidelineCorruptHFiles")) {
4157         sidelineCorruptHFiles = true;
4158       } else if (cmd.equals("-fixReferenceFiles")) {
4159         setFixReferenceFiles(true);
4160       } else if (cmd.equals("-fixEmptyMetaCells")) {
4161         setFixEmptyMetaCells(true);
4162       } else if (cmd.equals("-repair")) {
4163         // this attempts to merge overlapping hdfs regions, needs testing
4164         // under load
4165         setFixHdfsHoles(true);
4166         setFixHdfsOrphans(true);
4167         setFixMeta(true);
4168         setFixAssignments(true);
4169         setFixHdfsOverlaps(true);
4170         setFixVersionFile(true);
4171         setSidelineBigOverlaps(true);
4172         setFixSplitParents(false);
4173         setCheckHdfs(true);
4174         setFixReferenceFiles(true);
4175         setFixTableLocks(true);
4176       } else if (cmd.equals("-repairHoles")) {
4177         // this will make all missing hdfs regions available but may lose data
4178         setFixHdfsHoles(true);
4179         setFixHdfsOrphans(false);
4180         setFixMeta(true);
4181         setFixAssignments(true);
4182         setFixHdfsOverlaps(false);
4183         setSidelineBigOverlaps(false);
4184         setFixSplitParents(false);
4185         setCheckHdfs(true);
4186       } else if (cmd.equals("-maxOverlapsToSideline")) {
4187         if (i == args.length - 1) {
4188           errors.reportError(ERROR_CODE.WRONG_USAGE,
4189             "-maxOverlapsToSideline needs a numeric value argument.");
4190           return printUsageAndExit();
4191         }
4192         try {
4193           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4194           setMaxOverlapsToSideline(maxOverlapsToSideline);
4195         } catch (NumberFormatException e) {
4196           errors.reportError(ERROR_CODE.WRONG_USAGE,
4197             "-maxOverlapsToSideline needs a numeric value argument.");
4198           return printUsageAndExit();
4199         }
4200         i++;
4201       } else if (cmd.equals("-maxMerge")) {
4202         if (i == args.length - 1) {
4203           errors.reportError(ERROR_CODE.WRONG_USAGE,
4204             "-maxMerge needs a numeric value argument.");
4205           return printUsageAndExit();
4206         }
4207         try {
4208           int maxMerge = Integer.parseInt(args[i+1]);
4209           setMaxMerge(maxMerge);
4210         } catch (NumberFormatException e) {
4211           errors.reportError(ERROR_CODE.WRONG_USAGE,
4212             "-maxMerge needs a numeric value argument.");
4213           return printUsageAndExit();
4214         }
4215         i++;
4216       } else if (cmd.equals("-summary")) {
4217         setSummary();
4218       } else if (cmd.equals("-metaonly")) {
4219         setCheckMetaOnly();
4220       } else if (cmd.equals("-boundaries")) {
4221         setRegionBoundariesCheck();
4222       } else if (cmd.equals("-fixTableLocks")) {
4223         setFixTableLocks(true);
4224       } else if (cmd.startsWith("-")) {
4225         errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4226         return printUsageAndExit();
4227       } else {
4228         includeTable(TableName.valueOf(cmd));
4229         errors.print("Allow checking/fixes for table: " + cmd);
4230       }
4231     }
4232 
4233     errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));
4234 
4235     // pre-check current user has FS write permission or not
4236     try {
4237       preCheckPermission();
4238     } catch (AccessDeniedException ace) {
4239       Runtime.getRuntime().exit(-1);
4240     } catch (IOException ioe) {
4241       Runtime.getRuntime().exit(-1);
4242     }
4243 
4244     // do the real work of hbck
4245     connect();
4246 
4247     try {
4248       // if corrupt file mode is on, first fix them since they may be opened later
4249       if (checkCorruptHFiles || sidelineCorruptHFiles) {
4250         LOG.info("Checking all hfiles for corruption");
4251         HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4252         setHFileCorruptionChecker(hfcc); // so we can get result
4253         Collection<TableName> tables = getIncludedTables();
4254         Collection<Path> tableDirs = new ArrayList<Path>();
4255         Path rootdir = FSUtils.getRootDir(getConf());
4256         if (tables.size() > 0) {
4257           for (TableName t : tables) {
4258             tableDirs.add(FSUtils.getTableDir(rootdir, t));
4259           }
4260         } else {
4261           tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4262         }
4263         hfcc.checkTables(tableDirs);
4264         hfcc.report(errors);
4265       }
4266 
4267       // check and fix table integrity, region consistency.
4268       int code = onlineHbck();
4269       setRetCode(code);
4270       // If we have changed the HBase state it is better to run hbck again
4271       // to see if we haven't broken something else in the process.
4272       // We run it only once more because otherwise we can easily fall into
4273       // an infinite loop.
4274       if (shouldRerun()) {
4275         try {
4276           LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4277           Thread.sleep(sleepBeforeRerun);
4278         } catch (InterruptedException ie) {
4279           LOG.warn("Interrupted while sleeping");
4280           return this;
4281         }
4282         // Just report
4283         setFixAssignments(false);
4284         setFixMeta(false);
4285         setFixHdfsHoles(false);
4286         setFixHdfsOverlaps(false);
4287         setFixVersionFile(false);
4288         setFixTableOrphans(false);
4289         errors.resetErrors();
4290         code = onlineHbck();
4291         setRetCode(code);
4292       }
4293     } finally {
4294       IOUtils.cleanup(null, connection, meta, admin);
4295     }
4296     return this;
4297   }
4298 
4299   /**
4300    * ls -r for debugging purposes
4301    */
4302   void debugLsr(Path p) throws IOException {
4303     debugLsr(getConf(), p, errors);
4304   }
4305 
4306   /**
4307    * ls -r for debugging purposes
4308    */
4309   public static void debugLsr(Configuration conf,
4310       Path p) throws IOException {
4311     debugLsr(conf, p, new PrintingErrorReporter());
4312   }
4313 
4314   /**
4315    * ls -r for debugging purposes
4316    */
4317   public static void debugLsr(Configuration conf,
4318       Path p, ErrorReporter errors) throws IOException {
4319     if (!LOG.isDebugEnabled() || p == null) {
4320       return;
4321     }
4322     FileSystem fs = p.getFileSystem(conf);
4323 
4324     if (!fs.exists(p)) {
4325       // nothing
4326       return;
4327     }
4328     errors.print(p.toString());
4329 
4330     if (fs.isFile(p)) {
4331       return;
4332     }
4333 
4334     if (fs.getFileStatus(p).isDirectory()) {
4335       FileStatus[] fss= fs.listStatus(p);
4336       for (FileStatus status : fss) {
4337         debugLsr(conf, status.getPath(), errors);
4338       }
4339     }
4340   }
4341 }