View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.FileNotFoundException;
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.io.PrintWriter;
24  import java.io.StringWriter;
25  import java.net.URI;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.Collection;
29  import java.util.Collections;
30  import java.util.Comparator;
31  import java.util.HashMap;
32  import java.util.HashSet;
33  import java.util.Iterator;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.Map.Entry;
37  import java.util.Set;
38  import java.util.SortedMap;
39  import java.util.SortedSet;
40  import java.util.TreeMap;
41  import java.util.TreeSet;
42  import java.util.concurrent.Callable;
43  import java.util.concurrent.ConcurrentSkipListMap;
44  import java.util.concurrent.ExecutionException;
45  import java.util.concurrent.ExecutorService;
46  import java.util.concurrent.Future;
47  import java.util.concurrent.ScheduledThreadPoolExecutor;
48  import java.util.concurrent.atomic.AtomicInteger;
49  
50  import org.apache.commons.logging.Log;
51  import org.apache.commons.logging.LogFactory;
52  import org.apache.hadoop.classification.InterfaceAudience;
53  import org.apache.hadoop.classification.InterfaceStability;
54  import org.apache.hadoop.conf.Configuration;
55  import org.apache.hadoop.conf.Configured;
56  import org.apache.hadoop.fs.FileStatus;
57  import org.apache.hadoop.fs.FileSystem;
58  import org.apache.hadoop.fs.Path;
59  import org.apache.hadoop.fs.permission.FsAction;
60  import org.apache.hadoop.hbase.Abortable;
61  import org.apache.hadoop.hbase.Cell;
62  import org.apache.hadoop.hbase.ClusterStatus;
63  import org.apache.hadoop.hbase.HBaseConfiguration;
64  import org.apache.hadoop.hbase.HColumnDescriptor;
65  import org.apache.hadoop.hbase.HConstants;
66  import org.apache.hadoop.hbase.HRegionInfo;
67  import org.apache.hadoop.hbase.HRegionLocation;
68  import org.apache.hadoop.hbase.HTableDescriptor;
69  import org.apache.hadoop.hbase.KeyValue;
70  import org.apache.hadoop.hbase.MasterNotRunningException;
71  import org.apache.hadoop.hbase.RegionLocations;
72  import org.apache.hadoop.hbase.ServerName;
73  import org.apache.hadoop.hbase.TableName;
74  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
75  import org.apache.hadoop.hbase.MetaTableAccessor;
76  import org.apache.hadoop.hbase.client.Delete;
77  import org.apache.hadoop.hbase.client.Get;
78  import org.apache.hadoop.hbase.client.HBaseAdmin;
79  import org.apache.hadoop.hbase.client.HConnectable;
80  import org.apache.hadoop.hbase.client.HConnection;
81  import org.apache.hadoop.hbase.client.HConnectionManager;
82  import org.apache.hadoop.hbase.client.HTable;
83  import org.apache.hadoop.hbase.client.MetaScanner;
84  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
85  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
86  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
87  import org.apache.hadoop.hbase.client.Put;
88  import org.apache.hadoop.hbase.client.Result;
89  import org.apache.hadoop.hbase.client.RowMutations;
90  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
91  import org.apache.hadoop.hbase.io.hfile.HFile;
92  import org.apache.hadoop.hbase.master.MasterFileSystem;
93  import org.apache.hadoop.hbase.master.RegionState;
94  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
95  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
96  import org.apache.hadoop.hbase.regionserver.HRegion;
97  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
98  import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
99  import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
100 import org.apache.hadoop.hbase.security.UserProvider;
101 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
102 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
103 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
104 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
105 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
106 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
107 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
108 import org.apache.hadoop.hbase.zookeeper.ZKTableStateClientSideReader;
109 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
110 import org.apache.hadoop.hbase.security.AccessDeniedException;
111 import org.apache.hadoop.io.IOUtils;
112 import org.apache.hadoop.security.UserGroupInformation;
113 import org.apache.hadoop.util.ReflectionUtils;
114 import org.apache.hadoop.util.Tool;
115 import org.apache.hadoop.util.ToolRunner;
116 import org.apache.zookeeper.KeeperException;
117 
118 import com.google.common.base.Joiner;
119 import com.google.common.base.Preconditions;
120 import com.google.common.collect.Lists;
121 import com.google.common.collect.Multimap;
122 import com.google.common.collect.TreeMultimap;
123 import com.google.protobuf.ServiceException;
124 
125 /**
126  * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
127  * table integrity problems in a corrupted HBase.
128  * <p>
129  * Region consistency checks verify that hbase:meta, region deployment on region
130  * servers and the state of data in HDFS (.regioninfo files) all are in
131  * accordance.
132  * <p>
133  * Table integrity checks verify that all possible row keys resolve to exactly
134  * one region of a table.  This means there are no individual degenerate
135  * or backwards regions; no holes between regions; and that there are no
136  * overlapping regions.
137  * <p>
138  * The general repair strategy works in two phases:
139  * <ol>
140  * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
141  * <li> Repair Region Consistency with hbase:meta and assignments
142  * </ol>
143  * <p>
144  * For table integrity repairs, the tables' region directories are scanned
145  * for .regioninfo files.  Each table's integrity is then verified.  If there
146  * are any orphan regions (regions with no .regioninfo files) or holes, new
147  * regions are fabricated.  Backwards regions are sidelined as well as empty
148  * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
149  * a new region is created and all data is merged into the new region.
150  * <p>
151  * Table integrity repairs deal solely with HDFS and could potentially be done
152  * offline -- the hbase region servers or master do not need to be running.
153  * This phase can eventually be used to completely reconstruct the hbase:meta table in
154  * an offline fashion.
155  * <p>
156  * Region consistency requires three conditions -- 1) valid .regioninfo file
157  * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
158  * and 3) a region is deployed only at the regionserver that was assigned to
159  * with proper state in the master.
160  * <p>
161  * Region consistency repairs require hbase to be online so that hbck can
162  * contact the HBase master and region servers.  The hbck#connect() method must
163  * first be called successfully.  Much of the region consistency information
164  * is transient and less risky to repair.
165  * <p>
166  * If hbck is run from the command line, there are a handful of arguments that
167  * can be used to limit the kinds of repairs hbck will do.  See the code in
168  * {@link #printUsageAndExit()} for more details.
169  */
170 @InterfaceAudience.Public
171 @InterfaceStability.Evolving
172 public class HBaseFsck extends Configured {
173   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
174   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
175   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
176   private static boolean rsSupportsOffline = true;
177   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
178   private static final int DEFAULT_MAX_MERGE = 5;
179   private static final String TO_BE_LOADED = "to_be_loaded";
180 
181   /**********************
182    * Internal resources
183    **********************/
184   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
185   private ClusterStatus status;
186   private HConnection connection;
187   private HBaseAdmin admin;
188   private HTable meta;
189   // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
190   protected ExecutorService executor;
191   private long startMillis = System.currentTimeMillis();
192   private HFileCorruptionChecker hfcc;
193   private int retcode = 0;
194 
195   /***********
196    * Options
197    ***********/
198   private static boolean details = false; // do we display the full report
199   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
200   private boolean fixAssignments = false; // fix assignment errors?
201   private boolean fixMeta = false; // fix meta errors?
202   private boolean checkHdfs = true; // load and check fs consistency?
203   private boolean fixHdfsHoles = false; // fix fs holes?
204   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
205   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
206   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
207   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
208   private boolean fixSplitParents = false; // fix lingering split parents
209   private boolean fixReferenceFiles = false; // fix lingering reference store file
210   private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
211   private boolean fixTableLocks = false; // fix table locks which are expired
212 
213   // limit checking/fixes to listed tables, if empty attempt to check/fix all
214   // hbase:meta are always checked
215   private Set<TableName> tablesIncluded = new HashSet<TableName>();
216   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
217   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
218   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
219   private Path sidelineDir = null;
220 
221   private boolean rerun = false; // if we tried to fix something, rerun hbck
222   private static boolean summary = false; // if we want to print less output
223   private boolean checkMetaOnly = false;
224   private boolean checkRegionBoundaries = false;
225   private boolean ignorePreCheckPermission = false; // if pre-check permission
226 
227   /*********
228    * State
229    *********/
230   final private ErrorReporter errors;
231   int fixes = 0;
232 
233   /**
234    * This map contains the state of all hbck items.  It maps from encoded region
235    * name to HbckInfo structure.  The information contained in HbckInfo is used
236    * to detect and correct consistency (hdfs/meta/deployment) problems.
237    */
238   private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
239   private TreeSet<TableName> disabledTables =
240     new TreeSet<TableName>();
241   // Empty regioninfo qualifiers in hbase:meta
242   private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
243 
244   /**
245    * This map from Tablename -> TableInfo contains the structures necessary to
246    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
247    * to prevent dupes.
248    *
249    * If tablesIncluded is empty, this map contains all tables.
250    * Otherwise, it contains only meta tables and tables in tablesIncluded,
251    * unless checkMetaOnly is specified, in which case, it contains only
252    * the meta table
253    */
254   private SortedMap<TableName, TableInfo> tablesInfo =
255       new ConcurrentSkipListMap<TableName, TableInfo>();
256 
257   /**
258    * When initially looking at HDFS, we attempt to find any orphaned data.
259    */
260   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
261 
262   private Map<TableName, Set<String>> orphanTableDirs =
263       new HashMap<TableName, Set<String>>();
264 
265   /**
266    * Constructor
267    *
268    * @param conf Configuration object
269    * @throws MasterNotRunningException if the master is not running
270    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
271    */
272   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
273       ZooKeeperConnectionException, IOException, ClassNotFoundException {
274     super(conf);
275     // make a copy, just to be sure we're not overriding someone else's config
276     setConf(HBaseConfiguration.create(getConf()));
277     // disable blockcache for tool invocation, see HBASE-10500
278     getConf().setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0);
279     errors = getErrorReporter(conf);
280 
281     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
282     executor = new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
283   }
284 
285   /**
286    * Constructor
287    *
288    * @param conf
289    *          Configuration object
290    * @throws MasterNotRunningException
291    *           if the master is not running
292    * @throws ZooKeeperConnectionException
293    *           if unable to connect to ZooKeeper
294    */
295   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
296       ZooKeeperConnectionException, IOException, ClassNotFoundException {
297     super(conf);
298     errors = getErrorReporter(getConf());
299     this.executor = exec;
300   }
301 
302   /**
303    * To repair region consistency, one must call connect() in order to repair
304    * online state.
305    */
306   public void connect() throws IOException {
307     connection = HConnectionManager.createConnection(getConf());
308     admin = new HBaseAdmin(connection);
309     meta = new HTable(TableName.META_TABLE_NAME, connection);
310     status = admin.getClusterStatus();
311   }
312 
313   /**
314    * Get deployed regions according to the region servers.
315    */
316   private void loadDeployedRegions() throws IOException, InterruptedException {
317     // From the master, get a list of all known live region servers
318     Collection<ServerName> regionServers = status.getServers();
319     errors.print("Number of live region servers: " + regionServers.size());
320     if (details) {
321       for (ServerName rsinfo: regionServers) {
322         errors.print("  " + rsinfo.getServerName());
323       }
324     }
325 
326     // From the master, get a list of all dead region servers
327     Collection<ServerName> deadRegionServers = status.getDeadServerNames();
328     errors.print("Number of dead region servers: " + deadRegionServers.size());
329     if (details) {
330       for (ServerName name: deadRegionServers) {
331         errors.print("  " + name);
332       }
333     }
334 
335     // Print the current master name and state
336     errors.print("Master: " + status.getMaster());
337 
338     // Print the list of all backup masters
339     Collection<ServerName> backupMasters = status.getBackupMasters();
340     errors.print("Number of backup masters: " + backupMasters.size());
341     if (details) {
342       for (ServerName name: backupMasters) {
343         errors.print("  " + name);
344       }
345     }
346 
347     errors.print("Average load: " + status.getAverageLoad());
348     errors.print("Number of requests: " + status.getRequestsCount());
349     errors.print("Number of regions: " + status.getRegionsCount());
350 
351     Map<String, RegionState> rits = status.getRegionsInTransition();
352     errors.print("Number of regions in transition: " + rits.size());
353     if (details) {
354       for (RegionState state: rits.values()) {
355         errors.print("  " + state.toDescriptiveString());
356       }
357     }
358 
359     // Determine what's deployed
360     processRegionServers(regionServers);
361   }
362 
363   /**
364    * Clear the current state of hbck.
365    */
366   private void clearState() {
367     // Make sure regionInfo is empty before starting
368     fixes = 0;
369     regionInfoMap.clear();
370     emptyRegionInfoQualifiers.clear();
371     disabledTables.clear();
372     errors.clear();
373     tablesInfo.clear();
374     orphanHdfsDirs.clear();
375   }
376 
377   /**
378    * This repair method analyzes hbase data in hdfs and repairs it to satisfy
379    * the table integrity rules.  HBase doesn't need to be online for this
380    * operation to work.
381    */
382   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
383     // Initial pass to fix orphans.
384     if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
385         || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
386       LOG.info("Loading regioninfos HDFS");
387       // if nothing is happening this should always complete in two iterations.
388       int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
389       int curIter = 0;
390       do {
391         clearState(); // clears hbck state and reset fixes to 0 and.
392         // repair what's on HDFS
393         restoreHdfsIntegrity();
394         curIter++;// limit the number of iterations.
395       } while (fixes > 0 && curIter <= maxIterations);
396 
397       // Repairs should be done in the first iteration and verification in the second.
398       // If there are more than 2 passes, something funny has happened.
399       if (curIter > 2) {
400         if (curIter == maxIterations) {
401           LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
402               + "Tables integrity may not be fully repaired!");
403         } else {
404           LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
405         }
406       }
407     }
408   }
409 
410   /**
411    * This repair method requires the cluster to be online since it contacts
412    * region servers and the masters.  It makes each region's state in HDFS, in
413    * hbase:meta, and deployments consistent.
414    *
415    * @return If > 0 , number of errors detected, if < 0 there was an unrecoverable
416    * error.  If 0, we have a clean hbase.
417    */
418   public int onlineConsistencyRepair() throws IOException, KeeperException,
419     InterruptedException {
420     clearState();
421 
422     // get regions according to what is online on each RegionServer
423     loadDeployedRegions();
424     // check whether hbase:meta is deployed and online
425     recordMetaRegion();
426     // Check if hbase:meta is found only once and in the right place
427     if (!checkMetaRegion()) {
428       String errorMsg = "hbase:meta table is not consistent. ";
429       if (shouldFixAssignments()) {
430         errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
431       } else {
432         errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
433       }
434       errors.reportError(errorMsg + " Exiting...");
435       return -2;
436     }
437     // Not going with further consistency check for tables when hbase:meta itself is not consistent.
438     LOG.info("Loading regionsinfo from the hbase:meta table");
439     boolean success = loadMetaEntries();
440     if (!success) return -1;
441 
442     // Empty cells in hbase:meta?
443     reportEmptyMetaCells();
444 
445     // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
446     if (shouldFixEmptyMetaCells()) {
447       fixEmptyMetaCells();
448     }
449 
450     // get a list of all tables that have not changed recently.
451     if (!checkMetaOnly) {
452       reportTablesInFlux();
453     }
454 
455     // load regiondirs and regioninfos from HDFS
456     if (shouldCheckHdfs()) {
457       loadHdfsRegionDirs();
458       loadHdfsRegionInfos();
459     }
460 
461     // Get disabled tables from ZooKeeper
462     loadDisabledTables();
463 
464     // fix the orphan tables
465     fixOrphanTables();
466 
467     // Check and fix consistency
468     checkAndFixConsistency();
469 
470     // Check integrity (does not fix)
471     checkIntegrity();
472     return errors.getErrorList().size();
473   }
474 
475   /**
476    * Contacts the master and prints out cluster-wide information
477    * @return 0 on success, non-zero on failure
478    */
479   public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
480     // print hbase server version
481     errors.print("Version: " + status.getHBaseVersion());
482     offlineHdfsIntegrityRepair();
483 
484     // turn the balancer off
485     boolean oldBalancer = admin.setBalancerRunning(false, true);
486     try {
487       onlineConsistencyRepair();
488     }
489     finally {
490       admin.setBalancerRunning(oldBalancer, false);
491     }
492 
493     if (checkRegionBoundaries) {
494       checkRegionBoundaries();
495     }
496 
497     offlineReferenceFileRepair();
498 
499     checkAndFixTableLocks();
500 
501     // Print table summary
502     printTableSummary(tablesInfo);
503     return errors.summarize();
504   }
505 
506   public static byte[] keyOnly (byte[] b) {
507     if (b == null)
508       return b;
509     int rowlength = Bytes.toShort(b, 0);
510     byte[] result = new byte[rowlength];
511     System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
512     return result;
513   }
514 
515   private static class RegionBoundariesInformation {
516     public byte [] regionName;
517     public byte [] metaFirstKey;
518     public byte [] metaLastKey;
519     public byte [] storesFirstKey;
520     public byte [] storesLastKey;
521     @Override
522     public String toString () {
523       return "regionName=" + Bytes.toStringBinary(regionName) +
524              "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
525              "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
526              "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
527              "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
528     }
529   }
530 
531   public void checkRegionBoundaries() {
532     try {
533       ByteArrayComparator comparator = new ByteArrayComparator();
534       List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), false);
535       final RegionBoundariesInformation currentRegionBoundariesInformation =
536           new RegionBoundariesInformation();
537       Path hbaseRoot = FSUtils.getRootDir(getConf());
538       for (HRegionInfo regionInfo : regions) {
539         Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
540         currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
541         // For each region, get the start and stop key from the META and compare them to the
542         // same information from the Stores.
543         Path path = new Path(tableDir, regionInfo.getEncodedName());
544         FileSystem fs = path.getFileSystem(getConf());
545         FileStatus[] files = fs.listStatus(path);
546         // For all the column families in this region...
547         byte[] storeFirstKey = null;
548         byte[] storeLastKey = null;
549         for (FileStatus file : files) {
550           String fileName = file.getPath().toString();
551           fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
552           if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
553             FileStatus[] storeFiles = fs.listStatus(file.getPath());
554             // For all the stores in this column family.
555             for (FileStatus storeFile : storeFiles) {
556               HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
557                   getConf()), getConf());
558               if ((reader.getFirstKey() != null)
559                   && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
560                       reader.getFirstKey()) > 0))) {
561                 storeFirstKey = reader.getFirstKey();
562               }
563               if ((reader.getLastKey() != null)
564                   && ((storeLastKey == null) || (comparator.compare(storeLastKey,
565                       reader.getLastKey())) < 0)) {
566                 storeLastKey = reader.getLastKey();
567               }
568               reader.close();
569             }
570           }
571         }
572         currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
573         currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
574         currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
575         currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
576         if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
577           currentRegionBoundariesInformation.metaFirstKey = null;
578         if (currentRegionBoundariesInformation.metaLastKey.length == 0)
579           currentRegionBoundariesInformation.metaLastKey = null;
580 
581         // For a region to be correct, we need the META start key to be smaller or equal to the
582         // smallest start key from all the stores, and the start key from the next META entry to
583         // be bigger than the last key from all the current stores. First region start key is null;
584         // Last region end key is null; some regions can be empty and not have any store.
585 
586         boolean valid = true;
587         // Checking start key.
588         if ((currentRegionBoundariesInformation.storesFirstKey != null)
589             && (currentRegionBoundariesInformation.metaFirstKey != null)) {
590           valid = valid
591               && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
592                 currentRegionBoundariesInformation.metaFirstKey) >= 0;
593         }
594         // Checking stop key.
595         if ((currentRegionBoundariesInformation.storesLastKey != null)
596             && (currentRegionBoundariesInformation.metaLastKey != null)) {
597           valid = valid
598               && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
599                 currentRegionBoundariesInformation.metaLastKey) < 0;
600         }
601         if (!valid) {
602           errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
603             tablesInfo.get(regionInfo.getTable()));
604           LOG.warn("Region's boundaries not alligned between stores and META for:");
605           LOG.warn(currentRegionBoundariesInformation);
606         }
607       }
608     } catch (IOException e) {
609       LOG.error(e);
610     }
611   }
612 
613   /**
614    * Iterates through the list of all orphan/invalid regiondirs.
615    */
616   private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
617     for (HbckInfo hi : orphanHdfsDirs) {
618       LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
619       adoptHdfsOrphan(hi);
620     }
621   }
622 
623   /**
624    * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
625    * these orphans by creating a new region, and moving the column families,
626    * recovered edits, HLogs, into the new region dir.  We determine the region
627    * startkey and endkeys by looking at all of the hfiles inside the column
628    * families to identify the min and max keys. The resulting region will
629    * likely violate table integrity but will be dealt with by merging
630    * overlapping regions.
631    */
632   @SuppressWarnings("deprecation")
633   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
634     Path p = hi.getHdfsRegionDir();
635     FileSystem fs = p.getFileSystem(getConf());
636     FileStatus[] dirs = fs.listStatus(p);
637     if (dirs == null) {
638       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
639           p + ". This dir could probably be deleted.");
640       return ;
641     }
642 
643     TableName tableName = hi.getTableName();
644     TableInfo tableInfo = tablesInfo.get(tableName);
645     Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
646     HTableDescriptor template = tableInfo.getHTD();
647 
648     // find min and max key values
649     Pair<byte[],byte[]> orphanRegionRange = null;
650     for (FileStatus cf : dirs) {
651       String cfName= cf.getPath().getName();
652       // TODO Figure out what the special dirs are
653       if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
654 
655       FileStatus[] hfiles = fs.listStatus(cf.getPath());
656       for (FileStatus hfile : hfiles) {
657         byte[] start, end;
658         HFile.Reader hf = null;
659         try {
660           CacheConfig cacheConf = new CacheConfig(getConf());
661           hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
662           hf.loadFileInfo();
663           KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
664           start = startKv.getRow();
665           KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
666           end = endKv.getRow();
667         } catch (IOException ioe) {
668           LOG.warn("Problem reading orphan file " + hfile + ", skipping");
669           continue;
670         } catch (NullPointerException ioe) {
671           LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
672           continue;
673         } finally {
674           if (hf != null) {
675             hf.close();
676           }
677         }
678 
679         // expand the range to include the range of all hfiles
680         if (orphanRegionRange == null) {
681           // first range
682           orphanRegionRange = new Pair<byte[], byte[]>(start, end);
683         } else {
684           // TODO add test
685 
686           // expand range only if the hfile is wider.
687           if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
688             orphanRegionRange.setFirst(start);
689           }
690           if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
691             orphanRegionRange.setSecond(end);
692           }
693         }
694       }
695     }
696     if (orphanRegionRange == null) {
697       LOG.warn("No data in dir " + p + ", sidelining data");
698       fixes++;
699       sidelineRegionDir(fs, hi);
700       return;
701     }
702     LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
703         Bytes.toString(orphanRegionRange.getSecond()) + ")");
704 
705     // create new region on hdfs.  move data into place.
706     HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond());
707     LOG.info("Creating new region : " + hri);
708     HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
709     Path target = region.getRegionFileSystem().getRegionDir();
710 
711     // rename all the data to new region
712     mergeRegionDirs(target, hi);
713     fixes++;
714   }
715 
716   /**
717    * This method determines if there are table integrity errors in HDFS.  If
718    * there are errors and the appropriate "fix" options are enabled, the method
719    * will first correct orphan regions making them into legit regiondirs, and
720    * then reload to merge potentially overlapping regions.
721    *
722    * @return number of table integrity errors found
723    */
724   private int restoreHdfsIntegrity() throws IOException, InterruptedException {
725     // Determine what's on HDFS
726     LOG.info("Loading HBase regioninfo from HDFS...");
727     loadHdfsRegionDirs(); // populating regioninfo table.
728 
729     int errs = errors.getErrorList().size();
730     // First time just get suggestions.
731     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
732     checkHdfsIntegrity(false, false);
733 
734     if (errors.getErrorList().size() == errs) {
735       LOG.info("No integrity errors.  We are done with this phase. Glorious.");
736       return 0;
737     }
738 
739     if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
740       adoptHdfsOrphans(orphanHdfsDirs);
741       // TODO optimize by incrementally adding instead of reloading.
742     }
743 
744     // Make sure there are no holes now.
745     if (shouldFixHdfsHoles()) {
746       clearState(); // this also resets # fixes.
747       loadHdfsRegionDirs();
748       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
749       tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
750     }
751 
752     // Now we fix overlaps
753     if (shouldFixHdfsOverlaps()) {
754       // second pass we fix overlaps.
755       clearState(); // this also resets # fixes.
756       loadHdfsRegionDirs();
757       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
758       tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
759     }
760 
761     return errors.getErrorList().size();
762   }
763 
764   /**
765    * Scan all the store file names to find any lingering reference files,
766    * which refer to some none-exiting files. If "fix" option is enabled,
767    * any lingering reference file will be sidelined if found.
768    * <p>
769    * Lingering reference file prevents a region from opening. It has to
770    * be fixed before a cluster can start properly.
771    */
772   private void offlineReferenceFileRepair() throws IOException {
773     Configuration conf = getConf();
774     Path hbaseRoot = FSUtils.getRootDir(conf);
775     FileSystem fs = hbaseRoot.getFileSystem(conf);
776     Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot);
777     for (Path path: allFiles.values()) {
778       boolean isReference = false;
779       try {
780         isReference = StoreFileInfo.isReference(path);
781       } catch (Throwable t) {
782         // Ignore. Some files may not be store files at all.
783         // For example, files under .oldlogs folder in hbase:meta
784         // Warning message is already logged by
785         // StoreFile#isReference.
786       }
787       if (!isReference) continue;
788 
789       Path referredToFile = StoreFileInfo.getReferredToFile(path);
790       if (fs.exists(referredToFile)) continue;  // good, expected
791 
792       // Found a lingering reference file
793       errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
794         "Found lingering reference file " + path);
795       if (!shouldFixReferenceFiles()) continue;
796 
797       // Now, trying to fix it since requested
798       boolean success = false;
799       String pathStr = path.toString();
800 
801       // A reference file path should be like
802       // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
803       // Up 5 directories to get the root folder.
804       // So the file will be sidelined to a similar folder structure.
805       int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
806       for (int i = 0; index > 0 && i < 5; i++) {
807         index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
808       }
809       if (index > 0) {
810         Path rootDir = getSidelineDir();
811         Path dst = new Path(rootDir, pathStr.substring(index + 1));
812         fs.mkdirs(dst.getParent());
813         LOG.info("Trying to sildeline reference file "
814           + path + " to " + dst);
815         setShouldRerun();
816 
817         success = fs.rename(path, dst);
818       }
819       if (!success) {
820         LOG.error("Failed to sideline reference file " + path);
821       }
822     }
823   }
824 
825   /**
826    * TODO -- need to add tests for this.
827    */
828   private void reportEmptyMetaCells() {
829     errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
830       emptyRegionInfoQualifiers.size());
831     if (details) {
832       for (Result r: emptyRegionInfoQualifiers) {
833         errors.print("  " + r);
834       }
835     }
836   }
837 
838   /**
839    * TODO -- need to add tests for this.
840    */
841   private void reportTablesInFlux() {
842     AtomicInteger numSkipped = new AtomicInteger(0);
843     HTableDescriptor[] allTables = getTables(numSkipped);
844     errors.print("Number of Tables: " + allTables.length);
845     if (details) {
846       if (numSkipped.get() > 0) {
847         errors.detail("Number of Tables in flux: " + numSkipped.get());
848       }
849       for (HTableDescriptor td : allTables) {
850         errors.detail("  Table: " + td.getTableName() + "\t" +
851                            (td.isReadOnly() ? "ro" : "rw") + "\t" +
852                             (td.isMetaRegion() ? "META" : "    ") + "\t" +
853                            " families: " + td.getFamilies().size());
854       }
855     }
856   }
857 
858   public ErrorReporter getErrors() {
859     return errors;
860   }
861 
862   /**
863    * Read the .regioninfo file from the file system.  If there is no
864    * .regioninfo, add it to the orphan hdfs region list.
865    */
866   private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
867     Path regionDir = hbi.getHdfsRegionDir();
868     if (regionDir == null) {
869       LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
870       return;
871     }
872 
873     if (hbi.hdfsEntry.hri != null) {
874       // already loaded data
875       return;
876     }
877 
878     FileSystem fs = FileSystem.get(getConf());
879     HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
880     LOG.debug("HRegionInfo read: " + hri.toString());
881     hbi.hdfsEntry.hri = hri;
882   }
883 
884   /**
885    * Exception thrown when a integrity repair operation fails in an
886    * unresolvable way.
887    */
888   public static class RegionRepairException extends IOException {
889     private static final long serialVersionUID = 1L;
890     final IOException ioe;
891     public RegionRepairException(String s, IOException ioe) {
892       super(s);
893       this.ioe = ioe;
894     }
895   }
896 
897   /**
898    * Populate hbi's from regionInfos loaded from file system.
899    */
900   private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
901       throws IOException, InterruptedException {
902     tablesInfo.clear(); // regenerating the data
903     // generate region split structure
904     Collection<HbckInfo> hbckInfos = regionInfoMap.values();
905 
906     // Parallelized read of .regioninfo files.
907     List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
908     List<Future<Void>> hbiFutures;
909 
910     for (HbckInfo hbi : hbckInfos) {
911       WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
912       hbis.add(work);
913     }
914 
915     // Submit and wait for completion
916     hbiFutures = executor.invokeAll(hbis);
917 
918     for(int i=0; i<hbiFutures.size(); i++) {
919       WorkItemHdfsRegionInfo work = hbis.get(i);
920       Future<Void> f = hbiFutures.get(i);
921       try {
922         f.get();
923       } catch(ExecutionException e) {
924         LOG.warn("Failed to read .regioninfo file for region " +
925               work.hbi.getRegionNameAsString(), e.getCause());
926       }
927     }
928 
929     Path hbaseRoot = FSUtils.getRootDir(getConf());
930     FileSystem fs = hbaseRoot.getFileSystem(getConf());
931     // serialized table info gathering.
932     for (HbckInfo hbi: hbckInfos) {
933 
934       if (hbi.getHdfsHRI() == null) {
935         // was an orphan
936         continue;
937       }
938 
939 
940       // get table name from hdfs, populate various HBaseFsck tables.
941       TableName tableName = hbi.getTableName();
942       if (tableName == null) {
943         // There was an entry in hbase:meta not in the HDFS?
944         LOG.warn("tableName was null for: " + hbi);
945         continue;
946       }
947 
948       TableInfo modTInfo = tablesInfo.get(tableName);
949       if (modTInfo == null) {
950         // only executed once per table.
951         modTInfo = new TableInfo(tableName);
952         tablesInfo.put(tableName, modTInfo);
953         try {
954           HTableDescriptor htd =
955               FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
956           modTInfo.htds.add(htd);
957         } catch (IOException ioe) {
958           if (!orphanTableDirs.containsKey(tableName)) {
959             LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
960             //should only report once for each table
961             errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
962                 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
963             Set<String> columns = new HashSet<String>();
964             orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
965           }
966         }
967       }
968       if (!hbi.isSkipChecks()) {
969         modTInfo.addRegionInfo(hbi);
970       }
971     }
972 
973     loadTableInfosForTablesWithNoRegion();
974 
975     return tablesInfo;
976   }
977 
978   /**
979    * To get the column family list according to the column family dirs
980    * @param columns
981    * @param hbi
982    * @return a set of column families
983    * @throws IOException
984    */
985   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
986     Path regionDir = hbi.getHdfsRegionDir();
987     FileSystem fs = regionDir.getFileSystem(getConf());
988     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
989     for (FileStatus subdir : subDirs) {
990       String columnfamily = subdir.getPath().getName();
991       columns.add(columnfamily);
992     }
993     return columns;
994   }
995 
996   /**
997    * To fabricate a .tableinfo file with following contents<br>
998    * 1. the correct tablename <br>
999    * 2. the correct colfamily list<br>
1000    * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1001    * @throws IOException
1002    */
1003   private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1004       Set<String> columns) throws IOException {
1005     if (columns ==null || columns.isEmpty()) return false;
1006     HTableDescriptor htd = new HTableDescriptor(tableName);
1007     for (String columnfamimly : columns) {
1008       htd.addFamily(new HColumnDescriptor(columnfamimly));
1009     }
1010     fstd.createTableDescriptor(htd, true);
1011     return true;
1012   }
1013 
1014   /**
1015    * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1016    * @throws IOException
1017    */
1018   public void fixEmptyMetaCells() throws IOException {
1019     if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1020       LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1021       for (Result region : emptyRegionInfoQualifiers) {
1022         deleteMetaRegion(region.getRow());
1023         errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1024       }
1025       emptyRegionInfoQualifiers.clear();
1026     }
1027   }
1028 
1029   /**
1030    * To fix orphan table by creating a .tableinfo file under tableDir <br>
1031    * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1032    * 2. else create a default .tableinfo file with following items<br>
1033    * &nbsp;2.1 the correct tablename <br>
1034    * &nbsp;2.2 the correct colfamily list<br>
1035    * &nbsp;2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1036    * @throws IOException
1037    */
1038   public void fixOrphanTables() throws IOException {
1039     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1040 
1041       List<TableName> tmpList = new ArrayList<TableName>();
1042       tmpList.addAll(orphanTableDirs.keySet());
1043       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1044       Iterator<Entry<TableName, Set<String>>> iter =
1045           orphanTableDirs.entrySet().iterator();
1046       int j = 0;
1047       int numFailedCase = 0;
1048       FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1049       while (iter.hasNext()) {
1050         Entry<TableName, Set<String>> entry =
1051             (Entry<TableName, Set<String>>) iter.next();
1052         TableName tableName = entry.getKey();
1053         LOG.info("Trying to fix orphan table error: " + tableName);
1054         if (j < htds.length) {
1055           if (tableName.equals(htds[j].getTableName())) {
1056             HTableDescriptor htd = htds[j];
1057             LOG.info("fixing orphan table: " + tableName + " from cache");
1058             fstd.createTableDescriptor(htd, true);
1059             j++;
1060             iter.remove();
1061           }
1062         } else {
1063           if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1064             LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1065             LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1066             iter.remove();
1067           } else {
1068             LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1069             numFailedCase++;
1070           }
1071         }
1072         fixes++;
1073       }
1074 
1075       if (orphanTableDirs.isEmpty()) {
1076         // all orphanTableDirs are luckily recovered
1077         // re-run doFsck after recovering the .tableinfo file
1078         setShouldRerun();
1079         LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1080       } else if (numFailedCase > 0) {
1081         LOG.error("Failed to fix " + numFailedCase
1082             + " OrphanTables with default .tableinfo files");
1083       }
1084 
1085     }
1086     //cleanup the list
1087     orphanTableDirs.clear();
1088 
1089   }
1090 
1091   /**
1092    * This borrows code from MasterFileSystem.bootstrap()
1093    *
1094    * @return an open hbase:meta HRegion
1095    */
1096   private HRegion createNewMeta() throws IOException {
1097       Path rootdir = FSUtils.getRootDir(getConf());
1098     Configuration c = getConf();
1099     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1100     MasterFileSystem.setInfoFamilyCachingForMeta(false);
1101     HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c,
1102         HTableDescriptor.META_TABLEDESC);
1103     MasterFileSystem.setInfoFamilyCachingForMeta(true);
1104     return meta;
1105   }
1106 
1107   /**
1108    * Generate set of puts to add to new meta.  This expects the tables to be
1109    * clean with no overlaps or holes.  If there are any problems it returns null.
1110    *
1111    * @return An array list of puts to do in bulk, null if tables have problems
1112    */
1113   private ArrayList<Put> generatePuts(
1114       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1115     ArrayList<Put> puts = new ArrayList<Put>();
1116     boolean hasProblems = false;
1117     for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1118       TableName name = e.getKey();
1119 
1120       // skip "hbase:meta"
1121       if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1122         continue;
1123       }
1124 
1125       TableInfo ti = e.getValue();
1126       for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1127           .entrySet()) {
1128         Collection<HbckInfo> his = spl.getValue();
1129         int sz = his.size();
1130         if (sz != 1) {
1131           // problem
1132           LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1133               + " had " +  sz + " regions instead of exactly 1." );
1134           hasProblems = true;
1135           continue;
1136         }
1137 
1138         // add the row directly to meta.
1139         HbckInfo hi = his.iterator().next();
1140         HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
1141         Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1142         puts.add(p);
1143       }
1144     }
1145     return hasProblems ? null : puts;
1146   }
1147 
1148   /**
1149    * Suggest fixes for each table
1150    */
1151   private void suggestFixes(
1152       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1153     for (TableInfo tInfo : tablesInfo.values()) {
1154       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1155       tInfo.checkRegionChain(handler);
1156     }
1157   }
1158 
1159   /**
1160    * Rebuilds meta from information in hdfs/fs.  Depends on configuration
1161    * settings passed into hbck constructor to point to a particular fs/dir.
1162    *
1163    * @param fix flag that determines if method should attempt to fix holes
1164    * @return true if successful, false if attempt failed.
1165    */
1166   public boolean rebuildMeta(boolean fix) throws IOException,
1167       InterruptedException {
1168 
1169     // TODO check to make sure hbase is offline. (or at least the table
1170     // currently being worked on is off line)
1171 
1172     // Determine what's on HDFS
1173     LOG.info("Loading HBase regioninfo from HDFS...");
1174     loadHdfsRegionDirs(); // populating regioninfo table.
1175 
1176     int errs = errors.getErrorList().size();
1177     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1178     checkHdfsIntegrity(false, false);
1179 
1180     // make sure ok.
1181     if (errors.getErrorList().size() != errs) {
1182       // While in error state, iterate until no more fixes possible
1183       while(true) {
1184         fixes = 0;
1185         suggestFixes(tablesInfo);
1186         errors.clear();
1187         loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1188         checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1189 
1190         int errCount = errors.getErrorList().size();
1191 
1192         if (fixes == 0) {
1193           if (errCount > 0) {
1194             return false; // failed to fix problems.
1195           } else {
1196             break; // no fixes and no problems? drop out and fix stuff!
1197           }
1198         }
1199       }
1200     }
1201 
1202     // we can rebuild, move old meta out of the way and start
1203     LOG.info("HDFS regioninfo's seems good.  Sidelining old hbase:meta");
1204     Path backupDir = sidelineOldMeta();
1205 
1206     LOG.info("Creating new hbase:meta");
1207     HRegion meta = createNewMeta();
1208 
1209     // populate meta
1210     List<Put> puts = generatePuts(tablesInfo);
1211     if (puts == null) {
1212       LOG.fatal("Problem encountered when creating new hbase:meta entries.  " +
1213         "You may need to restore the previously sidelined hbase:meta");
1214       return false;
1215     }
1216     meta.batchMutate(puts.toArray(new Put[0]));
1217     HRegion.closeHRegion(meta);
1218     LOG.info("Success! hbase:meta table rebuilt.");
1219     LOG.info("Old hbase:meta is moved into " + backupDir);
1220     return true;
1221   }
1222 
1223   private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1224       boolean fixOverlaps) throws IOException {
1225     LOG.info("Checking HBase region split map from HDFS data...");
1226     for (TableInfo tInfo : tablesInfo.values()) {
1227       TableIntegrityErrorHandler handler;
1228       if (fixHoles || fixOverlaps) {
1229         handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1230           fixHoles, fixOverlaps);
1231       } else {
1232         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1233       }
1234       if (!tInfo.checkRegionChain(handler)) {
1235         // should dump info as well.
1236         errors.report("Found inconsistency in table " + tInfo.getName());
1237       }
1238     }
1239     return tablesInfo;
1240   }
1241 
1242   private Path getSidelineDir() throws IOException {
1243     if (sidelineDir == null) {
1244       Path hbaseDir = FSUtils.getRootDir(getConf());
1245       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1246       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1247           + startMillis);
1248     }
1249     return sidelineDir;
1250   }
1251 
1252   /**
1253    * Sideline a region dir (instead of deleting it)
1254    */
1255   Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1256     return sidelineRegionDir(fs, null, hi);
1257   }
1258 
1259   /**
1260    * Sideline a region dir (instead of deleting it)
1261    *
1262    * @param parentDir if specified, the region will be sidelined to
1263    * folder like .../parentDir/<table name>/<region name>. The purpose
1264    * is to group together similar regions sidelined, for example, those
1265    * regions should be bulk loaded back later on. If null, it is ignored.
1266    */
1267   Path sidelineRegionDir(FileSystem fs,
1268       String parentDir, HbckInfo hi) throws IOException {
1269     TableName tableName = hi.getTableName();
1270     Path regionDir = hi.getHdfsRegionDir();
1271 
1272     if (!fs.exists(regionDir)) {
1273       LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1274       return null;
1275     }
1276 
1277     Path rootDir = getSidelineDir();
1278     if (parentDir != null) {
1279       rootDir = new Path(rootDir, parentDir);
1280     }
1281     Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1282     Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1283     fs.mkdirs(sidelineRegionDir);
1284     boolean success = false;
1285     FileStatus[] cfs =  fs.listStatus(regionDir);
1286     if (cfs == null) {
1287       LOG.info("Region dir is empty: " + regionDir);
1288     } else {
1289       for (FileStatus cf : cfs) {
1290         Path src = cf.getPath();
1291         Path dst =  new Path(sidelineRegionDir, src.getName());
1292         if (fs.isFile(src)) {
1293           // simple file
1294           success = fs.rename(src, dst);
1295           if (!success) {
1296             String msg = "Unable to rename file " + src +  " to " + dst;
1297             LOG.error(msg);
1298             throw new IOException(msg);
1299           }
1300           continue;
1301         }
1302 
1303         // is a directory.
1304         fs.mkdirs(dst);
1305 
1306         LOG.info("Sidelining files from " + src + " into containing region " + dst);
1307         // FileSystem.rename is inconsistent with directories -- if the
1308         // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1309         // it moves the src into the dst dir resulting in (foo/a/b).  If
1310         // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1311         FileStatus[] hfiles = fs.listStatus(src);
1312         if (hfiles != null && hfiles.length > 0) {
1313           for (FileStatus hfile : hfiles) {
1314             success = fs.rename(hfile.getPath(), dst);
1315             if (!success) {
1316               String msg = "Unable to rename file " + src +  " to " + dst;
1317               LOG.error(msg);
1318               throw new IOException(msg);
1319             }
1320           }
1321         }
1322         LOG.debug("Sideline directory contents:");
1323         debugLsr(sidelineRegionDir);
1324       }
1325     }
1326 
1327     LOG.info("Removing old region dir: " + regionDir);
1328     success = fs.delete(regionDir, true);
1329     if (!success) {
1330       String msg = "Unable to delete dir " + regionDir;
1331       LOG.error(msg);
1332       throw new IOException(msg);
1333     }
1334     return sidelineRegionDir;
1335   }
1336 
1337   /**
1338    * Side line an entire table.
1339    */
1340   void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1341       Path backupHbaseDir) throws IOException {
1342     Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1343     if (fs.exists(tableDir)) {
1344       Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1345       fs.mkdirs(backupTableDir.getParent());
1346       boolean success = fs.rename(tableDir, backupTableDir);
1347       if (!success) {
1348         throw new IOException("Failed to move  " + tableName + " from "
1349             +  tableDir + " to " + backupTableDir);
1350       }
1351     } else {
1352       LOG.info("No previous " + tableName +  " exists.  Continuing.");
1353     }
1354   }
1355 
1356   /**
1357    * @return Path to backup of original directory
1358    */
1359   Path sidelineOldMeta() throws IOException {
1360     // put current hbase:meta aside.
1361     Path hbaseDir = FSUtils.getRootDir(getConf());
1362     FileSystem fs = hbaseDir.getFileSystem(getConf());
1363     Path backupDir = getSidelineDir();
1364     fs.mkdirs(backupDir);
1365 
1366     try {
1367       sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1368     } catch (IOException e) {
1369         LOG.fatal("... failed to sideline meta. Currently in inconsistent state.  To restore "
1370             + "try to rename hbase:meta in " + backupDir.getName() + " to "
1371             + hbaseDir.getName() + ".", e);
1372       throw e; // throw original exception
1373     }
1374     return backupDir;
1375   }
1376 
1377   /**
1378    * Load the list of disabled tables in ZK into local set.
1379    * @throws ZooKeeperConnectionException
1380    * @throws IOException
1381    */
1382   private void loadDisabledTables()
1383   throws ZooKeeperConnectionException, IOException {
1384     HConnectionManager.execute(new HConnectable<Void>(getConf()) {
1385       @Override
1386       public Void connect(HConnection connection) throws IOException {
1387         ZooKeeperWatcher zkw = createZooKeeperWatcher();
1388         try {
1389           for (TableName tableName :
1390               ZKTableStateClientSideReader.getDisabledOrDisablingTables(zkw)) {
1391             disabledTables.add(tableName);
1392           }
1393         } catch (KeeperException ke) {
1394           throw new IOException(ke);
1395         } catch (InterruptedException e) {
1396           throw new InterruptedIOException();
1397         } finally {
1398           zkw.close();
1399         }
1400         return null;
1401       }
1402     });
1403   }
1404 
1405   /**
1406    * Check if the specified region's table is disabled.
1407    */
1408   private boolean isTableDisabled(HRegionInfo regionInfo) {
1409     return disabledTables.contains(regionInfo.getTable());
1410   }
1411 
1412   /**
1413    * Scan HDFS for all regions, recording their information into
1414    * regionInfoMap
1415    */
1416   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1417     Path rootDir = FSUtils.getRootDir(getConf());
1418     FileSystem fs = rootDir.getFileSystem(getConf());
1419 
1420     // list all tables from HDFS
1421     List<FileStatus> tableDirs = Lists.newArrayList();
1422 
1423     boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1424 
1425     List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1426     for (Path path : paths) {
1427       TableName tableName = FSUtils.getTableName(path);
1428        if ((!checkMetaOnly &&
1429            isTableIncluded(tableName)) ||
1430            tableName.equals(TableName.META_TABLE_NAME)) {
1431          tableDirs.add(fs.getFileStatus(path));
1432        }
1433     }
1434 
1435     // verify that version file exists
1436     if (!foundVersionFile) {
1437       errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1438           "Version file does not exist in root dir " + rootDir);
1439       if (shouldFixVersionFile()) {
1440         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1441             + " file.");
1442         setShouldRerun();
1443         FSUtils.setVersion(fs, rootDir, getConf().getInt(
1444             HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1445             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1446             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1447       }
1448     }
1449 
1450     // level 1:  <HBASE_DIR>/*
1451     List<WorkItemHdfsDir> dirs = new ArrayList<WorkItemHdfsDir>(tableDirs.size());
1452     List<Future<Void>> dirsFutures;
1453 
1454     for (FileStatus tableDir : tableDirs) {
1455       LOG.debug("Loading region dirs from " +tableDir.getPath());
1456       dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir));
1457     }
1458 
1459     // Invoke and wait for Callables to complete
1460     dirsFutures = executor.invokeAll(dirs);
1461 
1462     for(Future<Void> f: dirsFutures) {
1463       try {
1464         f.get();
1465       } catch(ExecutionException e) {
1466         LOG.warn("Could not load region dir " , e.getCause());
1467       }
1468     }
1469   }
1470 
1471   /**
1472    * Record the location of the hbase:meta region as found in ZooKeeper.
1473    */
1474   private boolean recordMetaRegion() throws IOException {
1475     HRegionLocation metaLocation = connection.locateRegion(
1476       TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW);
1477 
1478     // Check if Meta region is valid and existing
1479     if (metaLocation == null || metaLocation.getRegionInfo() == null ||
1480         metaLocation.getHostname() == null) {
1481       errors.reportError(ERROR_CODE.NULL_META_REGION,
1482         "META region or some of its attributes are null.");
1483       return false;
1484     }
1485     ServerName sn;
1486     try {
1487       sn = getMetaRegionServerName();
1488     } catch (KeeperException e) {
1489       throw new IOException(e);
1490     }
1491     MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, System.currentTimeMillis());
1492     HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1493     if (hbckInfo == null) {
1494       regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1495     } else {
1496       hbckInfo.metaEntry = m;
1497     }
1498     return true;
1499   }
1500 
1501   private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1502     return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1503       @Override
1504       public void abort(String why, Throwable e) {
1505         LOG.error(why, e);
1506         System.exit(1);
1507       }
1508 
1509       @Override
1510       public boolean isAborted() {
1511         return false;
1512       }
1513 
1514     });
1515   }
1516 
1517   private ServerName getMetaRegionServerName()
1518   throws IOException, KeeperException {
1519     ZooKeeperWatcher zkw = createZooKeeperWatcher();
1520     ServerName sn = null;
1521     try {
1522       sn = new MetaTableLocator().getMetaRegionLocation(zkw);
1523     } finally {
1524       zkw.close();
1525     }
1526     return sn;
1527   }
1528 
1529   /**
1530    * Contacts each regionserver and fetches metadata about regions.
1531    * @param regionServerList - the list of region servers to connect to
1532    * @throws IOException if a remote or network exception occurs
1533    */
1534   void processRegionServers(Collection<ServerName> regionServerList)
1535     throws IOException, InterruptedException {
1536 
1537     List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1538     List<Future<Void>> workFutures;
1539 
1540     // loop to contact each region server in parallel
1541     for (ServerName rsinfo: regionServerList) {
1542       workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1543     }
1544 
1545     workFutures = executor.invokeAll(workItems);
1546 
1547     for(int i=0; i<workFutures.size(); i++) {
1548       WorkItemRegion item = workItems.get(i);
1549       Future<Void> f = workFutures.get(i);
1550       try {
1551         f.get();
1552       } catch(ExecutionException e) {
1553         LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1554             e.getCause());
1555       }
1556     }
1557   }
1558 
1559   /**
1560    * Check consistency of all regions that have been found in previous phases.
1561    */
1562   private void checkAndFixConsistency()
1563   throws IOException, KeeperException, InterruptedException {
1564 	  // Divide the checks in two phases. One for default/primary replicas and another
1565 	  // for the non-primary ones. Keeps code cleaner this way.
1566     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1567       if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1568         checkRegionConsistency(e.getKey(), e.getValue());
1569       }
1570     }
1571     boolean prevHdfsCheck = shouldCheckHdfs();
1572     setCheckHdfs(false); //replicas don't have any hdfs data
1573     // Run a pass over the replicas and fix any assignment issues that exist on the currently
1574     // deployed/undeployed replicas.
1575     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1576       if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1577         checkRegionConsistency(e.getKey(), e.getValue());
1578       }
1579     }
1580     setCheckHdfs(prevHdfsCheck);
1581   }
1582 
1583   private void preCheckPermission() throws IOException, AccessDeniedException {
1584     if (shouldIgnorePreCheckPermission()) {
1585       return;
1586     }
1587 
1588     Path hbaseDir = FSUtils.getRootDir(getConf());
1589     FileSystem fs = hbaseDir.getFileSystem(getConf());
1590     UserProvider userProvider = UserProvider.instantiate(getConf());
1591     UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1592     FileStatus[] files = fs.listStatus(hbaseDir);
1593     for (FileStatus file : files) {
1594       try {
1595         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1596       } catch (AccessDeniedException ace) {
1597         LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1598         errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1599           + " does not have write perms to " + file.getPath()
1600           + ". Please rerun hbck as hdfs user " + file.getOwner());
1601         throw ace;
1602       }
1603     }
1604   }
1605 
1606   /**
1607    * Deletes region from meta table
1608    */
1609   private void deleteMetaRegion(HbckInfo hi) throws IOException {
1610     deleteMetaRegion(hi.metaEntry.getRegionName());
1611   }
1612 
1613   /**
1614    * Deletes region from meta table
1615    */
1616   private void deleteMetaRegion(byte[] metaKey) throws IOException {
1617     Delete d = new Delete(metaKey);
1618     meta.delete(d);
1619     meta.flushCommits();
1620     LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1621   }
1622 
1623   /**
1624    * Reset the split parent region info in meta table
1625    */
1626   private void resetSplitParent(HbckInfo hi) throws IOException {
1627     RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
1628     Delete d = new Delete(hi.metaEntry.getRegionName());
1629     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1630     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1631     mutations.add(d);
1632 
1633     HRegionInfo hri = new HRegionInfo(hi.metaEntry);
1634     hri.setOffline(false);
1635     hri.setSplit(false);
1636     Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1637     mutations.add(p);
1638 
1639     meta.mutateRow(mutations);
1640     meta.flushCommits();
1641     LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
1642   }
1643 
1644   /**
1645    * This backwards-compatibility wrapper for permanently offlining a region
1646    * that should not be alive.  If the region server does not support the
1647    * "offline" method, it will use the closest unassign method instead.  This
1648    * will basically work until one attempts to disable or delete the affected
1649    * table.  The problem has to do with in-memory only master state, so
1650    * restarting the HMaster or failing over to another should fix this.
1651    */
1652   private void offline(byte[] regionName) throws IOException {
1653     String regionString = Bytes.toStringBinary(regionName);
1654     if (!rsSupportsOffline) {
1655       LOG.warn("Using unassign region " + regionString
1656           + " instead of using offline method, you should"
1657           + " restart HMaster after these repairs");
1658       admin.unassign(regionName, true);
1659       return;
1660     }
1661 
1662     // first time we assume the rs's supports #offline.
1663     try {
1664       LOG.info("Offlining region " + regionString);
1665       admin.offline(regionName);
1666     } catch (IOException ioe) {
1667       String notFoundMsg = "java.lang.NoSuchMethodException: " +
1668         "org.apache.hadoop.hbase.master.HMaster.offline([B)";
1669       if (ioe.getMessage().contains(notFoundMsg)) {
1670         LOG.warn("Using unassign region " + regionString
1671             + " instead of using offline method, you should"
1672             + " restart HMaster after these repairs");
1673         rsSupportsOffline = false; // in the future just use unassign
1674         admin.unassign(regionName, true);
1675         return;
1676       }
1677       throw ioe;
1678     }
1679   }
1680 
1681   private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
1682     undeployRegionsForHbi(hi);
1683     // undeploy replicas of the region (but only if the method is invoked for the primary)
1684     if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1685       return;
1686     }
1687     int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
1688     for (int i = 1; i < numReplicas; i++) {
1689       if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
1690       HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
1691           hi.getPrimaryHRIForDeployedReplica(), i);
1692       HbckInfo h = regionInfoMap.get(hri.getEncodedName());
1693       if (h != null) {
1694         undeployRegionsForHbi(h);
1695         //set skip checks; we undeployed it, and we don't want to evaluate this anymore
1696         //in consistency checks
1697         h.setSkipChecks(true);
1698       }
1699     }
1700   }
1701 
1702   private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
1703     for (OnlineEntry rse : hi.deployedEntries) {
1704       LOG.debug("Undeploy region "  + rse.hri + " from " + rse.hsa);
1705       try {
1706         HBaseFsckRepair.closeRegionSilentlyAndWait(admin, rse.hsa, rse.hri);
1707         offline(rse.hri.getRegionName());
1708       } catch (IOException ioe) {
1709         LOG.warn("Got exception when attempting to offline region "
1710             + Bytes.toString(rse.hri.getRegionName()), ioe);
1711       }
1712     }
1713   }
1714 
1715   /**
1716    * Attempts to undeploy a region from a region server based in information in
1717    * META.  Any operations that modify the file system should make sure that
1718    * its corresponding region is not deployed to prevent data races.
1719    *
1720    * A separate call is required to update the master in-memory region state
1721    * kept in the AssignementManager.  Because disable uses this state instead of
1722    * that found in META, we can't seem to cleanly disable/delete tables that
1723    * have been hbck fixed.  When used on a version of HBase that does not have
1724    * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master
1725    * restart or failover may be required.
1726    */
1727   private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
1728     if (hi.metaEntry == null && hi.hdfsEntry == null) {
1729       undeployRegions(hi);
1730       return;
1731     }
1732 
1733     // get assignment info and hregioninfo from meta.
1734     Get get = new Get(hi.getRegionName());
1735     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1736     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
1737     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
1738     // also get the locations of the replicas to close if the primary region is being closed
1739     if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1740       int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
1741       for (int i = 0; i < numReplicas; i++) {
1742         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
1743         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
1744       }
1745     }
1746     Result r = meta.get(get);
1747     RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
1748     if (rl == null) {
1749       LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
1750           " since meta does not have handle to reach it");
1751       return;
1752     }
1753     for (HRegionLocation h : rl.getRegionLocations()) {
1754       ServerName serverName = h.getServerName();
1755       if (serverName == null) {
1756         errors.reportError("Unable to close region "
1757             + hi.getRegionNameAsString() +  " because meta does not "
1758             + "have handle to reach it.");
1759         continue;
1760       }
1761       HRegionInfo hri = h.getRegionInfo();
1762       if (hri == null) {
1763         LOG.warn("Unable to close region " + hi.getRegionNameAsString()
1764             + " because hbase:meta had invalid or missing "
1765             + HConstants.CATALOG_FAMILY_STR + ":"
1766             + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
1767             + " qualifier value.");
1768         continue;
1769       }
1770       // close the region -- close files and remove assignment
1771       HBaseFsckRepair.closeRegionSilentlyAndWait(admin, serverName, hri);
1772     }
1773   }
1774 
1775   private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
1776     KeeperException, InterruptedException {
1777     // If we are trying to fix the errors
1778     if (shouldFixAssignments()) {
1779       errors.print(msg);
1780       undeployRegions(hbi);
1781       setShouldRerun();
1782       HRegionInfo hri = hbi.getHdfsHRI();
1783       if (hri == null) {
1784         hri = hbi.metaEntry;
1785       }
1786       HBaseFsckRepair.fixUnassigned(admin, hri);
1787       HBaseFsckRepair.waitUntilAssigned(admin, hri);
1788 
1789       // also assign replicas if needed (do it only when this call operates on a primary replica)
1790       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
1791       int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
1792       for (int i = 1; i < replicationCount; i++) {
1793         hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
1794         HbckInfo h = regionInfoMap.get(hri.getEncodedName());
1795         if (h != null) {
1796           undeployRegions(h);
1797           //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore
1798           //in consistency checks
1799           h.setSkipChecks(true);
1800         }
1801         HBaseFsckRepair.fixUnassigned(admin, hri);
1802         HBaseFsckRepair.waitUntilAssigned(admin, hri);
1803       }
1804 
1805     }
1806   }
1807 
1808   /**
1809    * Check a single region for consistency and correct deployment.
1810    */
1811   private void checkRegionConsistency(final String key, final HbckInfo hbi)
1812   throws IOException, KeeperException, InterruptedException {
1813 
1814 	if (hbi.isSkipChecks()) return;
1815 	String descriptiveName = hbi.toString();
1816     boolean inMeta = hbi.metaEntry != null;
1817     // In case not checking HDFS, assume the region is on HDFS
1818     boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
1819     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
1820     boolean isDeployed = !hbi.deployedOn.isEmpty();
1821     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
1822     boolean deploymentMatchesMeta =
1823       hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
1824       hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
1825     boolean splitParent =
1826       (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
1827     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
1828     boolean recentlyModified = inHdfs &&
1829       hbi.getModTime() + timelag > System.currentTimeMillis();
1830 
1831     // ========== First the healthy cases =============
1832     if (hbi.containsOnlyHdfsEdits()) {
1833       return;
1834     }
1835     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
1836       return;
1837     } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
1838       LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
1839         "tabled that is not deployed");
1840       return;
1841     } else if (recentlyModified) {
1842       LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
1843       return;
1844     }
1845     // ========== Cases where the region is not in hbase:meta =============
1846     else if (!inMeta && !inHdfs && !isDeployed) {
1847       // We shouldn't have record of this region at all then!
1848       assert false : "Entry for region with no data";
1849     } else if (!inMeta && !inHdfs && isDeployed) {
1850       errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
1851           + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
1852           "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1853       if (shouldFixAssignments()) {
1854         undeployRegions(hbi);
1855       }
1856 
1857     } else if (!inMeta && inHdfs && !isDeployed) {
1858       if (hbi.isMerged()) {
1859         // This region has already been merged, the remaining hdfs file will be
1860         // cleaned by CatalogJanitor later
1861         hbi.setSkipChecks(true);
1862         LOG.info("Region " + descriptiveName
1863             + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
1864         return;
1865       }
1866       errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
1867           + descriptiveName + " on HDFS, but not listed in hbase:meta " +
1868           "or deployed on any region server");
1869       // restore region consistency of an adopted orphan
1870       if (shouldFixMeta()) {
1871         if (!hbi.isHdfsRegioninfoPresent()) {
1872           LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
1873               +  " in table integrity repair phase if -fixHdfsOrphans was" +
1874               " used.");
1875           return;
1876         }
1877 
1878         LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
1879         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
1880         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
1881             admin.getClusterStatus().getServers(), numReplicas);
1882 
1883         tryAssignmentRepair(hbi, "Trying to reassign region...");
1884       }
1885 
1886     } else if (!inMeta && inHdfs && isDeployed) {
1887       errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
1888           + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1889       debugLsr(hbi.getHdfsRegionDir());
1890       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1891         // for replicas, this means that we should undeploy the region (we would have
1892         // gone over the primaries and fixed meta holes in first phase under
1893         // checkAndFixConsistency; we shouldn't get the condition !inMeta at
1894         // this stage unless unwanted replica)
1895         if (shouldFixAssignments()) {
1896           undeployRegionsForHbi(hbi);
1897         }
1898       }
1899       if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1900         if (!hbi.isHdfsRegioninfoPresent()) {
1901           LOG.error("This should have been repaired in table integrity repair phase");
1902           return;
1903         }
1904 
1905         LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
1906         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
1907         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
1908             admin.getClusterStatus().getServers(), numReplicas);
1909         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
1910       }
1911 
1912     // ========== Cases where the region is in hbase:meta =============
1913     } else if (inMeta && inHdfs && !isDeployed && splitParent) {
1914       // check whether this is an actual error, or just transient state where parent
1915       // is not cleaned
1916       if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
1917         // check that split daughters are there
1918         HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
1919         HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
1920         if (infoA != null && infoB != null) {
1921           // we already processed or will process daughters. Move on, nothing to see here.
1922           hbi.setSkipChecks(true);
1923           return;
1924         }
1925       }
1926       errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
1927           + descriptiveName + " is a split parent in META, in HDFS, "
1928           + "and not deployed on any region server. This could be transient.");
1929       if (shouldFixSplitParents()) {
1930         setShouldRerun();
1931         resetSplitParent(hbi);
1932       }
1933     } else if (inMeta && !inHdfs && !isDeployed) {
1934       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
1935           + descriptiveName + " found in META, but not in HDFS "
1936           + "or deployed on any region server.");
1937       if (shouldFixMeta()) {
1938         deleteMetaRegion(hbi);
1939       }
1940     } else if (inMeta && !inHdfs && isDeployed) {
1941       errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
1942           + " found in META, but not in HDFS, " +
1943           "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1944       // We treat HDFS as ground truth.  Any information in meta is transient
1945       // and equivalent data can be regenerated.  So, lets unassign and remove
1946       // these problems from META.
1947       if (shouldFixAssignments()) {
1948         errors.print("Trying to fix unassigned region...");
1949         undeployRegions(hbi);
1950       }
1951       if (shouldFixMeta()) {
1952         // wait for it to complete
1953         deleteMetaRegion(hbi);
1954       }
1955     } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
1956       errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
1957           + " not deployed on any region server.");
1958       tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
1959     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
1960       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
1961           "Region " + descriptiveName + " should not be deployed according " +
1962           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1963       if (shouldFixAssignments()) {
1964         errors.print("Trying to close the region " + descriptiveName);
1965         setShouldRerun();
1966         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
1967       }
1968     } else if (inMeta && inHdfs && isMultiplyDeployed) {
1969       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
1970           + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
1971           + " but is multiply assigned to region servers " +
1972           Joiner.on(", ").join(hbi.deployedOn));
1973       // If we are trying to fix the errors
1974       if (shouldFixAssignments()) {
1975         errors.print("Trying to fix assignment error...");
1976         setShouldRerun();
1977         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
1978       }
1979     } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
1980       errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
1981           + descriptiveName + " listed in hbase:meta on region server " +
1982           hbi.metaEntry.regionServer + " but found on region server " +
1983           hbi.deployedOn.get(0));
1984       // If we are trying to fix the errors
1985       if (shouldFixAssignments()) {
1986         errors.print("Trying to fix assignment error...");
1987         setShouldRerun();
1988         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
1989         HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
1990       }
1991     } else {
1992       errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
1993           " is in an unforeseen state:" +
1994           " inMeta=" + inMeta +
1995           " inHdfs=" + inHdfs +
1996           " isDeployed=" + isDeployed +
1997           " isMultiplyDeployed=" + isMultiplyDeployed +
1998           " deploymentMatchesMeta=" + deploymentMatchesMeta +
1999           " shouldBeDeployed=" + shouldBeDeployed);
2000     }
2001   }
2002 
2003   /**
2004    * Checks tables integrity. Goes over all regions and scans the tables.
2005    * Collects all the pieces for each table and checks if there are missing,
2006    * repeated or overlapping ones.
2007    * @throws IOException
2008    */
2009   SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2010     tablesInfo = new TreeMap<TableName,TableInfo> ();
2011     LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2012     for (HbckInfo hbi : regionInfoMap.values()) {
2013       // Check only valid, working regions
2014       if (hbi.metaEntry == null) {
2015         // this assumes that consistency check has run loadMetaEntry
2016         Path p = hbi.getHdfsRegionDir();
2017         if (p == null) {
2018           errors.report("No regioninfo in Meta or HDFS. " + hbi);
2019         }
2020 
2021         // TODO test.
2022         continue;
2023       }
2024       if (hbi.metaEntry.regionServer == null) {
2025         errors.detail("Skipping region because no region server: " + hbi);
2026         continue;
2027       }
2028       if (hbi.metaEntry.isOffline()) {
2029         errors.detail("Skipping region because it is offline: " + hbi);
2030         continue;
2031       }
2032       if (hbi.containsOnlyHdfsEdits()) {
2033         errors.detail("Skipping region because it only contains edits" + hbi);
2034         continue;
2035       }
2036 
2037       // Missing regionDir or over-deployment is checked elsewhere. Include
2038       // these cases in modTInfo, so we can evaluate those regions as part of
2039       // the region chain in META
2040       //if (hbi.foundRegionDir == null) continue;
2041       //if (hbi.deployedOn.size() != 1) continue;
2042       if (hbi.deployedOn.size() == 0) continue;
2043 
2044       // We should be safe here
2045       TableName tableName = hbi.metaEntry.getTable();
2046       TableInfo modTInfo = tablesInfo.get(tableName);
2047       if (modTInfo == null) {
2048         modTInfo = new TableInfo(tableName);
2049       }
2050       for (ServerName server : hbi.deployedOn) {
2051         modTInfo.addServer(server);
2052       }
2053 
2054       if (!hbi.isSkipChecks()) {
2055         modTInfo.addRegionInfo(hbi);
2056       }
2057 
2058       tablesInfo.put(tableName, modTInfo);
2059     }
2060 
2061     loadTableInfosForTablesWithNoRegion();
2062 
2063     for (TableInfo tInfo : tablesInfo.values()) {
2064       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2065       if (!tInfo.checkRegionChain(handler)) {
2066         errors.report("Found inconsistency in table " + tInfo.getName());
2067       }
2068     }
2069     return tablesInfo;
2070   }
2071 
2072   /** Loads table info's for tables that may not have been included, since there are no
2073    * regions reported for the table, but table dir is there in hdfs
2074    */
2075   private void loadTableInfosForTablesWithNoRegion() throws IOException {
2076     Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2077     for (HTableDescriptor htd : allTables.values()) {
2078       if (checkMetaOnly && !htd.isMetaTable()) {
2079         continue;
2080       }
2081 
2082       TableName tableName = htd.getTableName();
2083       if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2084         TableInfo tableInfo = new TableInfo(tableName);
2085         tableInfo.htds.add(htd);
2086         tablesInfo.put(htd.getTableName(), tableInfo);
2087       }
2088     }
2089   }
2090 
2091   /**
2092    * Merge hdfs data by moving from contained HbckInfo into targetRegionDir.
2093    * @return number of file move fixes done to merge regions.
2094    */
2095   public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2096     int fileMoves = 0;
2097     String thread = Thread.currentThread().getName();
2098     LOG.debug("[" + thread + "] Contained region dir after close and pause");
2099     debugLsr(contained.getHdfsRegionDir());
2100 
2101     // rename the contained into the container.
2102     FileSystem fs = targetRegionDir.getFileSystem(getConf());
2103     FileStatus[] dirs = null;
2104     try {
2105       dirs = fs.listStatus(contained.getHdfsRegionDir());
2106     } catch (FileNotFoundException fnfe) {
2107       // region we are attempting to merge in is not present!  Since this is a merge, there is
2108       // no harm skipping this region if it does not exist.
2109       if (!fs.exists(contained.getHdfsRegionDir())) {
2110         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2111             + " is missing. Assuming already sidelined or moved.");
2112       } else {
2113         sidelineRegionDir(fs, contained);
2114       }
2115       return fileMoves;
2116     }
2117 
2118     if (dirs == null) {
2119       if (!fs.exists(contained.getHdfsRegionDir())) {
2120         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2121             + " already sidelined.");
2122       } else {
2123         sidelineRegionDir(fs, contained);
2124       }
2125       return fileMoves;
2126     }
2127 
2128     for (FileStatus cf : dirs) {
2129       Path src = cf.getPath();
2130       Path dst =  new Path(targetRegionDir, src.getName());
2131 
2132       if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2133         // do not copy the old .regioninfo file.
2134         continue;
2135       }
2136 
2137       if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2138         // do not copy the .oldlogs files
2139         continue;
2140       }
2141 
2142       LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2143       // FileSystem.rename is inconsistent with directories -- if the
2144       // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2145       // it moves the src into the dst dir resulting in (foo/a/b).  If
2146       // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2147       for (FileStatus hfile : fs.listStatus(src)) {
2148         boolean success = fs.rename(hfile.getPath(), dst);
2149         if (success) {
2150           fileMoves++;
2151         }
2152       }
2153       LOG.debug("[" + thread + "] Sideline directory contents:");
2154       debugLsr(targetRegionDir);
2155     }
2156 
2157     // if all success.
2158     sidelineRegionDir(fs, contained);
2159     LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2160         getSidelineDir());
2161     debugLsr(contained.getHdfsRegionDir());
2162 
2163     return fileMoves;
2164   }
2165 
2166 
2167   static class WorkItemOverlapMerge implements Callable<Void> {
2168     private TableIntegrityErrorHandler handler;
2169     Collection<HbckInfo> overlapgroup;
2170 
2171     WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2172       this.handler = handler;
2173       this.overlapgroup = overlapgroup;
2174     }
2175 
2176     @Override
2177     public Void call() throws Exception {
2178       handler.handleOverlapGroup(overlapgroup);
2179       return null;
2180     }
2181   };
2182 
2183 
2184   /**
2185    * Maintain information about a particular table.
2186    */
2187   public class TableInfo {
2188     TableName tableName;
2189     TreeSet <ServerName> deployedOn;
2190 
2191     // backwards regions
2192     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2193 
2194     // sidelined big overlapped regions
2195     final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2196 
2197     // region split calculator
2198     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2199 
2200     // Histogram of different HTableDescriptors found.  Ideally there is only one!
2201     final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2202 
2203     // key = start split, values = set of splits in problem group
2204     final Multimap<byte[], HbckInfo> overlapGroups =
2205       TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2206 
2207     TableInfo(TableName name) {
2208       this.tableName = name;
2209       deployedOn = new TreeSet <ServerName>();
2210     }
2211 
2212     /**
2213      * @return descriptor common to all regions.  null if are none or multiple!
2214      */
2215     private HTableDescriptor getHTD() {
2216       if (htds.size() == 1) {
2217         return (HTableDescriptor)htds.toArray()[0];
2218       } else {
2219         LOG.error("None/Multiple table descriptors found for table '"
2220           + tableName + "' regions: " + htds);
2221       }
2222       return null;
2223     }
2224 
2225     public void addRegionInfo(HbckInfo hir) {
2226       if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2227         // end key is absolute end key, just add it.
2228         // ignore replicas other than primary for these checks
2229         if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2230         return;
2231       }
2232 
2233       // if not the absolute end key, check for cycle
2234       if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2235         errors.reportError(
2236             ERROR_CODE.REGION_CYCLE,
2237             String.format("The endkey for this region comes before the "
2238                 + "startkey, startkey=%s, endkey=%s",
2239                 Bytes.toStringBinary(hir.getStartKey()),
2240                 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2241         backwards.add(hir);
2242         return;
2243       }
2244 
2245       // main case, add to split calculator
2246       // ignore replicas other than primary for these checks
2247       if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2248     }
2249 
2250     public void addServer(ServerName server) {
2251       this.deployedOn.add(server);
2252     }
2253 
2254     public TableName getName() {
2255       return tableName;
2256     }
2257 
2258     public int getNumRegions() {
2259       return sc.getStarts().size() + backwards.size();
2260     }
2261 
2262     private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2263       ErrorReporter errors;
2264 
2265       IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2266         this.errors = errors;
2267         setTableInfo(ti);
2268       }
2269 
2270       @Override
2271       public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2272         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2273             "First region should start with an empty key.  You need to "
2274             + " create a new region and regioninfo in HDFS to plug the hole.",
2275             getTableInfo(), hi);
2276       }
2277 
2278       @Override
2279       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2280         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2281             "Last region should end with an empty key. You need to "
2282                 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2283       }
2284 
2285       @Override
2286       public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2287         errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2288             "Region has the same start and end key.", getTableInfo(), hi);
2289       }
2290 
2291       @Override
2292       public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2293         byte[] key = r1.getStartKey();
2294         // dup start key
2295         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2296             "Multiple regions have the same startkey: "
2297             + Bytes.toStringBinary(key), getTableInfo(), r1);
2298         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2299             "Multiple regions have the same startkey: "
2300             + Bytes.toStringBinary(key), getTableInfo(), r2);
2301       }
2302 
2303       @Override
2304       public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2305         errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2306             "There is an overlap in the region chain.",
2307             getTableInfo(), hi1, hi2);
2308       }
2309 
2310       @Override
2311       public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2312         errors.reportError(
2313             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2314             "There is a hole in the region chain between "
2315                 + Bytes.toStringBinary(holeStart) + " and "
2316                 + Bytes.toStringBinary(holeStop)
2317                 + ".  You need to create a new .regioninfo and region "
2318                 + "dir in hdfs to plug the hole.");
2319       }
2320     };
2321 
2322     /**
2323      * This handler fixes integrity errors from hdfs information.  There are
2324      * basically three classes of integrity problems 1) holes, 2) overlaps, and
2325      * 3) invalid regions.
2326      *
2327      * This class overrides methods that fix holes and the overlap group case.
2328      * Individual cases of particular overlaps are handled by the general
2329      * overlap group merge repair case.
2330      *
2331      * If hbase is online, this forces regions offline before doing merge
2332      * operations.
2333      */
2334     private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2335       Configuration conf;
2336 
2337       boolean fixOverlaps = true;
2338 
2339       HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2340           boolean fixHoles, boolean fixOverlaps) {
2341         super(ti, errors);
2342         this.conf = conf;
2343         this.fixOverlaps = fixOverlaps;
2344         // TODO properly use fixHoles
2345       }
2346 
2347       /**
2348        * This is a special case hole -- when the first region of a table is
2349        * missing from META, HBase doesn't acknowledge the existance of the
2350        * table.
2351        */
2352       @Override
2353       public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2354         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2355             "First region should start with an empty key.  Creating a new " +
2356             "region and regioninfo in HDFS to plug the hole.",
2357             getTableInfo(), next);
2358         HTableDescriptor htd = getTableInfo().getHTD();
2359         // from special EMPTY_START_ROW to next region's startKey
2360         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2361             HConstants.EMPTY_START_ROW, next.getStartKey());
2362 
2363         // TODO test
2364         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2365         LOG.info("Table region start key was not empty.  Created new empty region: "
2366             + newRegion + " " +region);
2367         fixes++;
2368       }
2369 
2370       @Override
2371       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2372         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2373             "Last region should end with an empty key.  Creating a new "
2374                 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2375         HTableDescriptor htd = getTableInfo().getHTD();
2376         // from curEndKey to EMPTY_START_ROW
2377         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2378             HConstants.EMPTY_START_ROW);
2379 
2380         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2381         LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
2382             + " " + region);
2383         fixes++;
2384       }
2385 
2386       /**
2387        * There is a hole in the hdfs regions that violates the table integrity
2388        * rules.  Create a new empty region that patches the hole.
2389        */
2390       @Override
2391       public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2392         errors.reportError(
2393             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2394             "There is a hole in the region chain between "
2395                 + Bytes.toStringBinary(holeStartKey) + " and "
2396                 + Bytes.toStringBinary(holeStopKey)
2397                 + ".  Creating a new regioninfo and region "
2398                 + "dir in hdfs to plug the hole.");
2399         HTableDescriptor htd = getTableInfo().getHTD();
2400         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2401         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2402         LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2403         fixes++;
2404       }
2405 
2406       /**
2407        * This takes set of overlapping regions and merges them into a single
2408        * region.  This covers cases like degenerate regions, shared start key,
2409        * general overlaps, duplicate ranges, and partial overlapping regions.
2410        *
2411        * Cases:
2412        * - Clean regions that overlap
2413        * - Only .oldlogs regions (can't find start/stop range, or figure out)
2414        *
2415        * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
2416        */
2417       @Override
2418       public void handleOverlapGroup(Collection<HbckInfo> overlap)
2419           throws IOException {
2420         Preconditions.checkNotNull(overlap);
2421         Preconditions.checkArgument(overlap.size() >0);
2422 
2423         if (!this.fixOverlaps) {
2424           LOG.warn("Not attempting to repair overlaps.");
2425           return;
2426         }
2427 
2428         if (overlap.size() > maxMerge) {
2429           LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2430             "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2431           if (sidelineBigOverlaps) {
2432             // we only sideline big overlapped groups that exceeds the max number of regions to merge
2433             sidelineBigOverlaps(overlap);
2434           }
2435           return;
2436         }
2437 
2438         mergeOverlaps(overlap);
2439       }
2440 
2441       void mergeOverlaps(Collection<HbckInfo> overlap)
2442           throws IOException {
2443         String thread = Thread.currentThread().getName();
2444         LOG.info("== [" + thread + "] Merging regions into one region: "
2445           + Joiner.on(",").join(overlap));
2446         // get the min / max range and close all concerned regions
2447         Pair<byte[], byte[]> range = null;
2448         for (HbckInfo hi : overlap) {
2449           if (range == null) {
2450             range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2451           } else {
2452             if (RegionSplitCalculator.BYTES_COMPARATOR
2453                 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2454               range.setFirst(hi.getStartKey());
2455             }
2456             if (RegionSplitCalculator.BYTES_COMPARATOR
2457                 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2458               range.setSecond(hi.getEndKey());
2459             }
2460           }
2461           // need to close files so delete can happen.
2462           LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
2463           LOG.debug("[" + thread + "] Contained region dir before close");
2464           debugLsr(hi.getHdfsRegionDir());
2465           try {
2466             LOG.info("[" + thread + "] Closing region: " + hi);
2467             closeRegion(hi);
2468           } catch (IOException ioe) {
2469             LOG.warn("[" + thread + "] Was unable to close region " + hi
2470               + ".  Just continuing... ", ioe);
2471           } catch (InterruptedException e) {
2472             LOG.warn("[" + thread + "] Was unable to close region " + hi
2473               + ".  Just continuing... ", e);
2474           }
2475 
2476           try {
2477             LOG.info("[" + thread + "] Offlining region: " + hi);
2478             offline(hi.getRegionName());
2479           } catch (IOException ioe) {
2480             LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2481               + ".  Just continuing... ", ioe);
2482           }
2483         }
2484 
2485         // create new empty container region.
2486         HTableDescriptor htd = getTableInfo().getHTD();
2487         // from start key to end Key
2488         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
2489             range.getSecond());
2490         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2491         LOG.info("[" + thread + "] Created new empty container region: " +
2492             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
2493         debugLsr(region.getRegionFileSystem().getRegionDir());
2494 
2495         // all target regions are closed, should be able to safely cleanup.
2496         boolean didFix= false;
2497         Path target = region.getRegionFileSystem().getRegionDir();
2498         for (HbckInfo contained : overlap) {
2499           LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
2500           int merges = mergeRegionDirs(target, contained);
2501           if (merges > 0) {
2502             didFix = true;
2503           }
2504         }
2505         if (didFix) {
2506           fixes++;
2507         }
2508       }
2509 
2510       /**
2511        * Sideline some regions in a big overlap group so that it
2512        * will have fewer regions, and it is easier to merge them later on.
2513        *
2514        * @param bigOverlap the overlapped group with regions more than maxMerge
2515        * @throws IOException
2516        */
2517       void sidelineBigOverlaps(
2518           Collection<HbckInfo> bigOverlap) throws IOException {
2519         int overlapsToSideline = bigOverlap.size() - maxMerge;
2520         if (overlapsToSideline > maxOverlapsToSideline) {
2521           overlapsToSideline = maxOverlapsToSideline;
2522         }
2523         List<HbckInfo> regionsToSideline =
2524           RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
2525         FileSystem fs = FileSystem.get(conf);
2526         for (HbckInfo regionToSideline: regionsToSideline) {
2527           try {
2528             LOG.info("Closing region: " + regionToSideline);
2529             closeRegion(regionToSideline);
2530           } catch (IOException ioe) {
2531             LOG.warn("Was unable to close region " + regionToSideline
2532               + ".  Just continuing... ", ioe);
2533           } catch (InterruptedException e) {
2534             LOG.warn("Was unable to close region " + regionToSideline
2535               + ".  Just continuing... ", e);
2536           }
2537 
2538           try {
2539             LOG.info("Offlining region: " + regionToSideline);
2540             offline(regionToSideline.getRegionName());
2541           } catch (IOException ioe) {
2542             LOG.warn("Unable to offline region from master: " + regionToSideline
2543               + ".  Just continuing... ", ioe);
2544           }
2545 
2546           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
2547           Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
2548           if (sidelineRegionDir != null) {
2549             sidelinedRegions.put(sidelineRegionDir, regionToSideline);
2550             LOG.info("After sidelined big overlapped region: "
2551               + regionToSideline.getRegionNameAsString()
2552               + " to " + sidelineRegionDir.toString());
2553             fixes++;
2554           }
2555         }
2556       }
2557     }
2558 
2559     /**
2560      * Check the region chain (from META) of this table.  We are looking for
2561      * holes, overlaps, and cycles.
2562      * @return false if there are errors
2563      * @throws IOException
2564      */
2565     public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
2566       // When table is disabled no need to check for the region chain. Some of the regions
2567       // accidently if deployed, this below code might report some issues like missing start
2568       // or end regions or region hole in chain and may try to fix which is unwanted.
2569       if (disabledTables.contains(this.tableName)) {
2570         return true;
2571       }
2572       int originalErrorsCount = errors.getErrorList().size();
2573       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
2574       SortedSet<byte[]> splits = sc.getSplits();
2575 
2576       byte[] prevKey = null;
2577       byte[] problemKey = null;
2578 
2579       if (splits.size() == 0) {
2580         // no region for this table
2581         handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
2582       }
2583 
2584       for (byte[] key : splits) {
2585         Collection<HbckInfo> ranges = regions.get(key);
2586         if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
2587           for (HbckInfo rng : ranges) {
2588             handler.handleRegionStartKeyNotEmpty(rng);
2589           }
2590         }
2591 
2592         // check for degenerate ranges
2593         for (HbckInfo rng : ranges) {
2594           // special endkey case converts '' to null
2595           byte[] endKey = rng.getEndKey();
2596           endKey = (endKey.length == 0) ? null : endKey;
2597           if (Bytes.equals(rng.getStartKey(),endKey)) {
2598             handler.handleDegenerateRegion(rng);
2599           }
2600         }
2601 
2602         if (ranges.size() == 1) {
2603           // this split key is ok -- no overlap, not a hole.
2604           if (problemKey != null) {
2605             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2606           }
2607           problemKey = null; // fell through, no more problem.
2608         } else if (ranges.size() > 1) {
2609           // set the new problem key group name, if already have problem key, just
2610           // keep using it.
2611           if (problemKey == null) {
2612             // only for overlap regions.
2613             LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
2614             problemKey = key;
2615           }
2616           overlapGroups.putAll(problemKey, ranges);
2617 
2618           // record errors
2619           ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
2620           //  this dumb and n^2 but this shouldn't happen often
2621           for (HbckInfo r1 : ranges) {
2622             if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2623             subRange.remove(r1);
2624             for (HbckInfo r2 : subRange) {
2625               if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2626               if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
2627                 handler.handleDuplicateStartKeys(r1,r2);
2628               } else {
2629                 // overlap
2630                 handler.handleOverlapInRegionChain(r1, r2);
2631               }
2632             }
2633           }
2634 
2635         } else if (ranges.size() == 0) {
2636           if (problemKey != null) {
2637             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2638           }
2639           problemKey = null;
2640 
2641           byte[] holeStopKey = sc.getSplits().higher(key);
2642           // if higher key is null we reached the top.
2643           if (holeStopKey != null) {
2644             // hole
2645             handler.handleHoleInRegionChain(key, holeStopKey);
2646           }
2647         }
2648         prevKey = key;
2649       }
2650 
2651       // When the last region of a table is proper and having an empty end key, 'prevKey'
2652       // will be null.
2653       if (prevKey != null) {
2654         handler.handleRegionEndKeyNotEmpty(prevKey);
2655       }
2656 
2657       // TODO fold this into the TableIntegrityHandler
2658       if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
2659         LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
2660             " false to run serially.");
2661         boolean ok = handleOverlapsParallel(handler, prevKey);
2662         if (!ok) {
2663           return false;
2664         }
2665       } else {
2666         LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
2667             " true to run in parallel.");
2668         for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2669           handler.handleOverlapGroup(overlap);
2670         }
2671       }
2672 
2673       if (details) {
2674         // do full region split map dump
2675         errors.print("---- Table '"  +  this.tableName
2676             + "': region split map");
2677         dump(splits, regions);
2678         errors.print("---- Table '"  +  this.tableName
2679             + "': overlap groups");
2680         dumpOverlapProblems(overlapGroups);
2681         errors.print("There are " + overlapGroups.keySet().size()
2682             + " overlap groups with " + overlapGroups.size()
2683             + " overlapping regions");
2684       }
2685       if (!sidelinedRegions.isEmpty()) {
2686         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
2687         errors.print("---- Table '"  +  this.tableName
2688             + "': sidelined big overlapped regions");
2689         dumpSidelinedRegions(sidelinedRegions);
2690       }
2691       return errors.getErrorList().size() == originalErrorsCount;
2692     }
2693 
2694     private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
2695         throws IOException {
2696       // we parallelize overlap handler for the case we have lots of groups to fix.  We can
2697       // safely assume each group is independent.
2698       List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
2699       List<Future<Void>> rets;
2700       for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2701         //
2702         merges.add(new WorkItemOverlapMerge(overlap, handler));
2703       }
2704       try {
2705         rets = executor.invokeAll(merges);
2706       } catch (InterruptedException e) {
2707         LOG.error("Overlap merges were interrupted", e);
2708         return false;
2709       }
2710       for(int i=0; i<merges.size(); i++) {
2711         WorkItemOverlapMerge work = merges.get(i);
2712         Future<Void> f = rets.get(i);
2713         try {
2714           f.get();
2715         } catch(ExecutionException e) {
2716           LOG.warn("Failed to merge overlap group" + work, e.getCause());
2717         } catch (InterruptedException e) {
2718           LOG.error("Waiting for overlap merges was interrupted", e);
2719           return false;
2720         }
2721       }
2722       return true;
2723     }
2724 
2725     /**
2726      * This dumps data in a visually reasonable way for visual debugging
2727      *
2728      * @param splits
2729      * @param regions
2730      */
2731     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
2732       // we display this way because the last end key should be displayed as well.
2733       StringBuilder sb = new StringBuilder();
2734       for (byte[] k : splits) {
2735         sb.setLength(0); // clear out existing buffer, if any.
2736         sb.append(Bytes.toStringBinary(k) + ":\t");
2737         for (HbckInfo r : regions.get(k)) {
2738           sb.append("[ "+ r.toString() + ", "
2739               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
2740         }
2741         errors.print(sb.toString());
2742       }
2743     }
2744   }
2745 
2746   public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
2747     // we display this way because the last end key should be displayed as
2748     // well.
2749     for (byte[] k : regions.keySet()) {
2750       errors.print(Bytes.toStringBinary(k) + ":");
2751       for (HbckInfo r : regions.get(k)) {
2752         errors.print("[ " + r.toString() + ", "
2753             + Bytes.toStringBinary(r.getEndKey()) + "]");
2754       }
2755       errors.print("----");
2756     }
2757   }
2758 
2759   public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
2760     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
2761       TableName tableName = entry.getValue().getTableName();
2762       Path path = entry.getKey();
2763       errors.print("This sidelined region dir should be bulk loaded: "
2764         + path.toString());
2765       errors.print("Bulk load command looks like: "
2766         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
2767         + path.toUri().getPath() + " "+ tableName);
2768     }
2769   }
2770 
2771   public Multimap<byte[], HbckInfo> getOverlapGroups(
2772       TableName table) {
2773     TableInfo ti = tablesInfo.get(table);
2774     return ti.overlapGroups;
2775   }
2776 
2777   /**
2778    * Return a list of user-space table names whose metadata have not been
2779    * modified in the last few milliseconds specified by timelag
2780    * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
2781    * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
2782    * milliseconds specified by timelag, then the table is a candidate to be returned.
2783    * @return tables that have not been modified recently
2784    * @throws IOException if an error is encountered
2785    */
2786   HTableDescriptor[] getTables(AtomicInteger numSkipped) {
2787     List<TableName> tableNames = new ArrayList<TableName>();
2788     long now = System.currentTimeMillis();
2789 
2790     for (HbckInfo hbi : regionInfoMap.values()) {
2791       MetaEntry info = hbi.metaEntry;
2792 
2793       // if the start key is zero, then we have found the first region of a table.
2794       // pick only those tables that were not modified in the last few milliseconds.
2795       if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
2796         if (info.modTime + timelag < now) {
2797           tableNames.add(info.getTable());
2798         } else {
2799           numSkipped.incrementAndGet(); // one more in-flux table
2800         }
2801       }
2802     }
2803     return getHTableDescriptors(tableNames);
2804   }
2805 
2806   HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
2807     HTableDescriptor[] htd = new HTableDescriptor[0];
2808     HBaseAdmin admin = null;
2809     try {
2810       LOG.info("getHTableDescriptors == tableNames => " + tableNames);
2811       admin = new HBaseAdmin(getConf());
2812       htd = admin.getTableDescriptorsByTableName(tableNames);
2813     } catch (IOException e) {
2814       LOG.debug("Exception getting table descriptors", e);
2815     } finally {
2816       if (admin != null) {
2817         try {
2818           admin.close();
2819         } catch (IOException e) {
2820           LOG.debug("Exception closing HBaseAdmin", e);
2821         }
2822       }
2823     }
2824     return htd;
2825   }
2826 
2827   /**
2828    * Gets the entry in regionInfo corresponding to the the given encoded
2829    * region name. If the region has not been seen yet, a new entry is added
2830    * and returned.
2831    */
2832   private synchronized HbckInfo getOrCreateInfo(String name) {
2833     HbckInfo hbi = regionInfoMap.get(name);
2834     if (hbi == null) {
2835       hbi = new HbckInfo(null);
2836       regionInfoMap.put(name, hbi);
2837     }
2838     return hbi;
2839   }
2840 
2841   private void checkAndFixTableLocks() throws IOException {
2842     TableLockChecker checker = new TableLockChecker(createZooKeeperWatcher(), errors);
2843     checker.checkTableLocks();
2844 
2845     if (this.fixTableLocks) {
2846       checker.fixExpiredTableLocks();
2847     }
2848   }
2849 
2850   /**
2851     * Check values in regionInfo for hbase:meta
2852     * Check if zero or more than one regions with hbase:meta are found.
2853     * If there are inconsistencies (i.e. zero or more than one regions
2854     * pretend to be holding the hbase:meta) try to fix that and report an error.
2855     * @throws IOException from HBaseFsckRepair functions
2856    * @throws KeeperException
2857    * @throws InterruptedException
2858     */
2859   boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
2860     List<HbckInfo> metaRegions = Lists.newArrayList();
2861     for (HbckInfo value : regionInfoMap.values()) {
2862       if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
2863         metaRegions.add(value);
2864       }
2865     }
2866 
2867     // There will be always one entry in regionInfoMap corresponding to hbase:meta
2868     // Check the deployed servers. It should be exactly one server.
2869     List<ServerName> servers = new ArrayList<ServerName>();
2870     HbckInfo metaHbckInfo = null;
2871     if (!metaRegions.isEmpty()) {
2872       metaHbckInfo = metaRegions.get(0);
2873       servers = metaHbckInfo.deployedOn;
2874     }
2875     if (servers.size() != 1) {
2876       if (servers.size() == 0) {
2877         errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta is not found on any region.");
2878         if (shouldFixAssignments()) {
2879           errors.print("Trying to fix a problem with hbase:meta..");
2880           setShouldRerun();
2881           // try to fix it (treat it as unassigned region)
2882           HBaseFsckRepair.fixUnassigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
2883           HBaseFsckRepair.waitUntilAssigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
2884         }
2885       } else if (servers.size() > 1) {
2886         errors
2887             .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta is found on more than one region.");
2888         if (shouldFixAssignments()) {
2889           if (metaHbckInfo == null) {
2890             errors.print(
2891               "Unable to fix problem with hbase:meta due to hbase:meta region info missing");
2892             return false;
2893           }
2894           errors.print("Trying to fix a problem with hbase:meta..");
2895           setShouldRerun();
2896           // try fix it (treat is a dupe assignment)
2897           HBaseFsckRepair.fixMultiAssignment(admin, metaHbckInfo.metaEntry, servers);
2898         }
2899       }
2900       // rerun hbck with hopefully fixed META
2901       return false;
2902     }
2903     // no errors, so continue normally
2904     return true;
2905   }
2906 
2907   /**
2908    * Scan hbase:meta, adding all regions found to the regionInfo map.
2909    * @throws IOException if an error is encountered
2910    */
2911   boolean loadMetaEntries() throws IOException {
2912     MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
2913       int countRecord = 1;
2914 
2915       // comparator to sort KeyValues with latest modtime
2916       final Comparator<Cell> comp = new Comparator<Cell>() {
2917         @Override
2918         public int compare(Cell k1, Cell k2) {
2919           return (int)(k1.getTimestamp() - k2.getTimestamp());
2920         }
2921       };
2922 
2923       @Override
2924       public boolean processRow(Result result) throws IOException {
2925         try {
2926 
2927           // record the latest modification of this META record
2928           long ts =  Collections.max(result.listCells(), comp).getTimestamp();
2929           RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
2930           if (rl == null) {
2931             emptyRegionInfoQualifiers.add(result);
2932             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
2933               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
2934             return true;
2935           }
2936           ServerName sn = null;
2937           if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
2938               rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
2939             emptyRegionInfoQualifiers.add(result);
2940             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
2941               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
2942             return true;
2943           }
2944           HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
2945           if (!(isTableIncluded(hri.getTable())
2946               || hri.isMetaRegion())) {
2947             return true;
2948           }
2949           PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
2950           for (HRegionLocation h : rl.getRegionLocations()) {
2951             if (h == null || h.getRegionInfo() == null) {
2952               continue;
2953             }
2954             sn = h.getServerName();
2955             hri = h.getRegionInfo();
2956 
2957             MetaEntry m = null;
2958             if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2959               m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
2960             } else {
2961               m = new MetaEntry(hri, sn, ts, null, null);
2962             }
2963             HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
2964             if (previous == null) {
2965               regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
2966             } else if (previous.metaEntry == null) {
2967               previous.metaEntry = m;
2968             } else {
2969               throw new IOException("Two entries in hbase:meta are same " + previous);
2970             }
2971           }
2972           PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
2973           for (HRegionInfo mergeRegion : new HRegionInfo[] {
2974               mergeRegions.getFirst(), mergeRegions.getSecond() }) {
2975             if (mergeRegion != null) {
2976               // This region is already been merged
2977               HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
2978               hbInfo.setMerged(true);
2979             }
2980           }
2981 
2982           // show proof of progress to the user, once for every 100 records.
2983           if (countRecord % 100 == 0) {
2984             errors.progress();
2985           }
2986           countRecord++;
2987           return true;
2988         } catch (RuntimeException e) {
2989           LOG.error("Result=" + result);
2990           throw e;
2991         }
2992       }
2993     };
2994     if (!checkMetaOnly) {
2995       // Scan hbase:meta to pick up user regions
2996       MetaScanner.metaScan(getConf(), visitor);
2997     }
2998 
2999     errors.print("");
3000     return true;
3001   }
3002 
3003   /**
3004    * Stores the regioninfo entries scanned from META
3005    */
3006   static class MetaEntry extends HRegionInfo {
3007     ServerName regionServer;   // server hosting this region
3008     long modTime;          // timestamp of most recent modification metadata
3009     HRegionInfo splitA, splitB; //split daughters
3010 
3011     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3012       this(rinfo, regionServer, modTime, null, null);
3013     }
3014 
3015     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3016         HRegionInfo splitA, HRegionInfo splitB) {
3017       super(rinfo);
3018       this.regionServer = regionServer;
3019       this.modTime = modTime;
3020       this.splitA = splitA;
3021       this.splitB = splitB;
3022     }
3023 
3024     @Override
3025     public boolean equals(Object o) {
3026       boolean superEq = super.equals(o);
3027       if (!superEq) {
3028         return superEq;
3029       }
3030 
3031       MetaEntry me = (MetaEntry) o;
3032       if (!regionServer.equals(me.regionServer)) {
3033         return false;
3034       }
3035       return (modTime == me.modTime);
3036     }
3037 
3038     @Override
3039     public int hashCode() {
3040       int hash = Arrays.hashCode(getRegionName());
3041       hash ^= getRegionId();
3042       hash ^= Arrays.hashCode(getStartKey());
3043       hash ^= Arrays.hashCode(getEndKey());
3044       hash ^= Boolean.valueOf(isOffline()).hashCode();
3045       hash ^= getTable().hashCode();
3046       if (regionServer != null) {
3047         hash ^= regionServer.hashCode();
3048       }
3049       hash ^= modTime;
3050       return hash;
3051     }
3052   }
3053 
3054   /**
3055    * Stores the regioninfo entries from HDFS
3056    */
3057   static class HdfsEntry {
3058     HRegionInfo hri;
3059     Path hdfsRegionDir = null;
3060     long hdfsRegionDirModTime  = 0;
3061     boolean hdfsRegioninfoFilePresent = false;
3062     boolean hdfsOnlyEdits = false;
3063   }
3064 
3065   /**
3066    * Stores the regioninfo retrieved from Online region servers.
3067    */
3068   static class OnlineEntry {
3069     HRegionInfo hri;
3070     ServerName hsa;
3071 
3072     @Override
3073     public String toString() {
3074       return hsa.toString() + ";" + hri.getRegionNameAsString();
3075     }
3076   }
3077 
3078   /**
3079    * Maintain information about a particular region.  It gathers information
3080    * from three places -- HDFS, META, and region servers.
3081    */
3082   public static class HbckInfo implements KeyRange {
3083     private MetaEntry metaEntry = null; // info in META
3084     private HdfsEntry hdfsEntry = null; // info in HDFS
3085     private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
3086     private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
3087     private boolean skipChecks = false; // whether to skip further checks to this region info.
3088     private boolean isMerged = false;// whether this region has already been merged into another one
3089     private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3090     private HRegionInfo primaryHRIForDeployedReplica = null;
3091 
3092     HbckInfo(MetaEntry metaEntry) {
3093       this.metaEntry = metaEntry;
3094     }
3095 
3096     public int getReplicaId() {
3097       if (metaEntry != null) return metaEntry.getReplicaId();
3098       return deployedReplicaId;
3099     }
3100 
3101     public synchronized void addServer(HRegionInfo hri, ServerName server) {
3102       OnlineEntry rse = new OnlineEntry() ;
3103       rse.hri = hri;
3104       rse.hsa = server;
3105       this.deployedEntries.add(rse);
3106       this.deployedOn.add(server);
3107       // save the replicaId that we see deployed in the cluster
3108       this.deployedReplicaId = hri.getReplicaId();
3109       this.primaryHRIForDeployedReplica =
3110           RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3111     }
3112 
3113     @Override
3114     public synchronized String toString() {
3115       StringBuilder sb = new StringBuilder();
3116       sb.append("{ meta => ");
3117       sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3118       sb.append( ", hdfs => " + getHdfsRegionDir());
3119       sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3120       sb.append( ", replicaId => " + getReplicaId());
3121       sb.append(" }");
3122       return sb.toString();
3123     }
3124 
3125     @Override
3126     public byte[] getStartKey() {
3127       if (this.metaEntry != null) {
3128         return this.metaEntry.getStartKey();
3129       } else if (this.hdfsEntry != null) {
3130         return this.hdfsEntry.hri.getStartKey();
3131       } else {
3132         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3133         return null;
3134       }
3135     }
3136 
3137     @Override
3138     public byte[] getEndKey() {
3139       if (this.metaEntry != null) {
3140         return this.metaEntry.getEndKey();
3141       } else if (this.hdfsEntry != null) {
3142         return this.hdfsEntry.hri.getEndKey();
3143       } else {
3144         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3145         return null;
3146       }
3147     }
3148 
3149     public TableName getTableName() {
3150       if (this.metaEntry != null) {
3151         return this.metaEntry.getTable();
3152       } else if (this.hdfsEntry != null) {
3153         // we are only guaranteed to have a path and not an HRI for hdfsEntry,
3154         // so we get the name from the Path
3155         Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3156         return FSUtils.getTableName(tableDir);
3157       } else {
3158         // return the info from the first online/deployed hri
3159         for (OnlineEntry e : deployedEntries) {
3160           return e.hri.getTable();
3161         }
3162         return null;
3163       }
3164     }
3165 
3166     public String getRegionNameAsString() {
3167       if (metaEntry != null) {
3168         return metaEntry.getRegionNameAsString();
3169       } else if (hdfsEntry != null) {
3170         if (hdfsEntry.hri != null) {
3171           return hdfsEntry.hri.getRegionNameAsString();
3172         }
3173       } else {
3174         // return the info from the first online/deployed hri
3175         for (OnlineEntry e : deployedEntries) {
3176           return e.hri.getRegionNameAsString();
3177         }
3178       }
3179       return null;
3180     }
3181 
3182     public byte[] getRegionName() {
3183       if (metaEntry != null) {
3184         return metaEntry.getRegionName();
3185       } else if (hdfsEntry != null) {
3186         return hdfsEntry.hri.getRegionName();
3187       } else {
3188         // return the info from the first online/deployed hri
3189         for (OnlineEntry e : deployedEntries) {
3190           return e.hri.getRegionName();
3191         }
3192         return null;
3193       }
3194     }
3195 
3196     public HRegionInfo getPrimaryHRIForDeployedReplica() {
3197       return primaryHRIForDeployedReplica;
3198     }
3199 
3200     Path getHdfsRegionDir() {
3201       if (hdfsEntry == null) {
3202         return null;
3203       }
3204       return hdfsEntry.hdfsRegionDir;
3205     }
3206 
3207     boolean containsOnlyHdfsEdits() {
3208       if (hdfsEntry == null) {
3209         return false;
3210       }
3211       return hdfsEntry.hdfsOnlyEdits;
3212     }
3213 
3214     boolean isHdfsRegioninfoPresent() {
3215       if (hdfsEntry == null) {
3216         return false;
3217       }
3218       return hdfsEntry.hdfsRegioninfoFilePresent;
3219     }
3220 
3221     long getModTime() {
3222       if (hdfsEntry == null) {
3223         return 0;
3224       }
3225       return hdfsEntry.hdfsRegionDirModTime;
3226     }
3227 
3228     HRegionInfo getHdfsHRI() {
3229       if (hdfsEntry == null) {
3230         return null;
3231       }
3232       return hdfsEntry.hri;
3233     }
3234 
3235     public void setSkipChecks(boolean skipChecks) {
3236       this.skipChecks = skipChecks;
3237     }
3238 
3239     public boolean isSkipChecks() {
3240       return skipChecks;
3241     }
3242 
3243     public void setMerged(boolean isMerged) {
3244       this.isMerged = isMerged;
3245     }
3246 
3247     public boolean isMerged() {
3248       return this.isMerged;
3249     }
3250   }
3251 
3252   final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3253     @Override
3254     public int compare(HbckInfo l, HbckInfo r) {
3255       if (l == r) {
3256         // same instance
3257         return 0;
3258       }
3259 
3260       int tableCompare = l.getTableName().compareTo(r.getTableName());
3261       if (tableCompare != 0) {
3262         return tableCompare;
3263       }
3264 
3265       int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3266           l.getStartKey(), r.getStartKey());
3267       if (startComparison != 0) {
3268         return startComparison;
3269       }
3270 
3271       // Special case for absolute endkey
3272       byte[] endKey = r.getEndKey();
3273       endKey = (endKey.length == 0) ? null : endKey;
3274       byte[] endKey2 = l.getEndKey();
3275       endKey2 = (endKey2.length == 0) ? null : endKey2;
3276       int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3277           endKey2,  endKey);
3278 
3279       if (endComparison != 0) {
3280         return endComparison;
3281       }
3282 
3283       // use regionId as tiebreaker.
3284       // Null is considered after all possible values so make it bigger.
3285       if (l.hdfsEntry == null && r.hdfsEntry == null) {
3286         return 0;
3287       }
3288       if (l.hdfsEntry == null && r.hdfsEntry != null) {
3289         return 1;
3290       }
3291       // l.hdfsEntry must not be null
3292       if (r.hdfsEntry == null) {
3293         return -1;
3294       }
3295       // both l.hdfsEntry and r.hdfsEntry must not be null.
3296       return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3297     }
3298   };
3299 
3300   /**
3301    * Prints summary of all tables found on the system.
3302    */
3303   private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3304     StringBuilder sb = new StringBuilder();
3305     errors.print("Summary:");
3306     for (TableInfo tInfo : tablesInfo.values()) {
3307       if (errors.tableHasErrors(tInfo)) {
3308         errors.print("Table " + tInfo.getName() + " is inconsistent.");
3309       } else {
3310         errors.print("  " + tInfo.getName() + " is okay.");
3311       }
3312       errors.print("    Number of regions: " + tInfo.getNumRegions());
3313       sb.setLength(0); // clear out existing buffer, if any.
3314       sb.append("    Deployed on: ");
3315       for (ServerName server : tInfo.deployedOn) {
3316         sb.append(" " + server.toString());
3317       }
3318       errors.print(sb.toString());
3319     }
3320   }
3321 
3322   static ErrorReporter getErrorReporter(
3323       final Configuration conf) throws ClassNotFoundException {
3324     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3325     return (ErrorReporter)ReflectionUtils.newInstance(reporter, conf);
3326   }
3327 
3328   public interface ErrorReporter {
3329     enum ERROR_CODE {
3330       UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3331       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
3332       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3333       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3334       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3335       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3336       WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR
3337     }
3338     void clear();
3339     void report(String message);
3340     void reportError(String message);
3341     void reportError(ERROR_CODE errorCode, String message);
3342     void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3343     void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3344     void reportError(
3345       ERROR_CODE errorCode,
3346       String message,
3347       TableInfo table,
3348       HbckInfo info1,
3349       HbckInfo info2
3350     );
3351     int summarize();
3352     void detail(String details);
3353     ArrayList<ERROR_CODE> getErrorList();
3354     void progress();
3355     void print(String message);
3356     void resetErrors();
3357     boolean tableHasErrors(TableInfo table);
3358   }
3359 
3360   static class PrintingErrorReporter implements ErrorReporter {
3361     public int errorCount = 0;
3362     private int showProgress;
3363 
3364     Set<TableInfo> errorTables = new HashSet<TableInfo>();
3365 
3366     // for use by unit tests to verify which errors were discovered
3367     private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3368 
3369     @Override
3370     public void clear() {
3371       errorTables.clear();
3372       errorList.clear();
3373       errorCount = 0;
3374     }
3375 
3376     @Override
3377     public synchronized void reportError(ERROR_CODE errorCode, String message) {
3378       if (errorCode == ERROR_CODE.WRONG_USAGE) {
3379         System.err.println(message);
3380         return;
3381       }
3382 
3383       errorList.add(errorCode);
3384       if (!summary) {
3385         System.out.println("ERROR: " + message);
3386       }
3387       errorCount++;
3388       showProgress = 0;
3389     }
3390 
3391     @Override
3392     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
3393       errorTables.add(table);
3394       reportError(errorCode, message);
3395     }
3396 
3397     @Override
3398     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3399                                          HbckInfo info) {
3400       errorTables.add(table);
3401       String reference = "(region " + info.getRegionNameAsString() + ")";
3402       reportError(errorCode, reference + " " + message);
3403     }
3404 
3405     @Override
3406     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3407                                          HbckInfo info1, HbckInfo info2) {
3408       errorTables.add(table);
3409       String reference = "(regions " + info1.getRegionNameAsString()
3410           + " and " + info2.getRegionNameAsString() + ")";
3411       reportError(errorCode, reference + " " + message);
3412     }
3413 
3414     @Override
3415     public synchronized void reportError(String message) {
3416       reportError(ERROR_CODE.UNKNOWN, message);
3417     }
3418 
3419     /**
3420      * Report error information, but do not increment the error count.  Intended for cases
3421      * where the actual error would have been reported previously.
3422      * @param message
3423      */
3424     @Override
3425     public synchronized void report(String message) {
3426       if (! summary) {
3427         System.out.println("ERROR: " + message);
3428       }
3429       showProgress = 0;
3430     }
3431 
3432     @Override
3433     public synchronized int summarize() {
3434       System.out.println(Integer.toString(errorCount) +
3435                          " inconsistencies detected.");
3436       if (errorCount == 0) {
3437         System.out.println("Status: OK");
3438         return 0;
3439       } else {
3440         System.out.println("Status: INCONSISTENT");
3441         return -1;
3442       }
3443     }
3444 
3445     @Override
3446     public ArrayList<ERROR_CODE> getErrorList() {
3447       return errorList;
3448     }
3449 
3450     @Override
3451     public synchronized void print(String message) {
3452       if (!summary) {
3453         System.out.println(message);
3454       }
3455     }
3456 
3457     @Override
3458     public boolean tableHasErrors(TableInfo table) {
3459       return errorTables.contains(table);
3460     }
3461 
3462     @Override
3463     public void resetErrors() {
3464       errorCount = 0;
3465     }
3466 
3467     @Override
3468     public synchronized void detail(String message) {
3469       if (details) {
3470         System.out.println(message);
3471       }
3472       showProgress = 0;
3473     }
3474 
3475     @Override
3476     public synchronized void progress() {
3477       if (showProgress++ == 10) {
3478         if (!summary) {
3479           System.out.print(".");
3480         }
3481         showProgress = 0;
3482       }
3483     }
3484   }
3485 
3486   /**
3487    * Contact a region server and get all information from it
3488    */
3489   static class WorkItemRegion implements Callable<Void> {
3490     private HBaseFsck hbck;
3491     private ServerName rsinfo;
3492     private ErrorReporter errors;
3493     private HConnection connection;
3494 
3495     WorkItemRegion(HBaseFsck hbck, ServerName info,
3496                    ErrorReporter errors, HConnection connection) {
3497       this.hbck = hbck;
3498       this.rsinfo = info;
3499       this.errors = errors;
3500       this.connection = connection;
3501     }
3502 
3503     @Override
3504     public synchronized Void call() throws IOException {
3505       errors.progress();
3506       try {
3507         BlockingInterface server = connection.getAdmin(rsinfo);
3508 
3509         // list all online regions from this region server
3510         List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
3511         regions = filterRegions(regions);
3512 
3513         if (details) {
3514           errors.detail("RegionServer: " + rsinfo.getServerName() +
3515                            " number of regions: " + regions.size());
3516           for (HRegionInfo rinfo: regions) {
3517             errors.detail("  " + rinfo.getRegionNameAsString() +
3518                              " id: " + rinfo.getRegionId() +
3519                              " encoded_name: " + rinfo.getEncodedName() +
3520                              " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
3521                              " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
3522           }
3523         }
3524 
3525         // check to see if the existence of this region matches the region in META
3526         for (HRegionInfo r:regions) {
3527           HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
3528           hbi.addServer(r, rsinfo);
3529         }
3530       } catch (IOException e) {          // unable to connect to the region server.
3531         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
3532           " Unable to fetch region information. " + e);
3533         throw e;
3534       }
3535       return null;
3536     }
3537 
3538     private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
3539       List<HRegionInfo> ret = Lists.newArrayList();
3540       for (HRegionInfo hri : regions) {
3541         if (hri.isMetaTable() || (!hbck.checkMetaOnly
3542             && hbck.isTableIncluded(hri.getTable()))) {
3543           ret.add(hri);
3544         }
3545       }
3546       return ret;
3547     }
3548   }
3549 
3550   /**
3551    * Contact hdfs and get all information about specified table directory into
3552    * regioninfo list.
3553    */
3554   static class WorkItemHdfsDir implements Callable<Void> {
3555     private HBaseFsck hbck;
3556     private FileStatus tableDir;
3557     private ErrorReporter errors;
3558     private FileSystem fs;
3559 
3560     WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
3561                     FileStatus status) {
3562       this.hbck = hbck;
3563       this.fs = fs;
3564       this.tableDir = status;
3565       this.errors = errors;
3566     }
3567 
3568     @Override
3569     public synchronized Void call() throws IOException {
3570       try {
3571         // level 2: <HBASE_DIR>/<table>/*
3572         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
3573         for (FileStatus regionDir : regionDirs) {
3574           String encodedName = regionDir.getPath().getName();
3575           // ignore directories that aren't hexadecimal
3576           if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
3577             continue;
3578           }
3579 
3580           LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
3581           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
3582           HdfsEntry he = new HdfsEntry();
3583           synchronized (hbi) {
3584             if (hbi.getHdfsRegionDir() != null) {
3585               errors.print("Directory " + encodedName + " duplicate??" +
3586                            hbi.getHdfsRegionDir());
3587             }
3588 
3589             he.hdfsRegionDir = regionDir.getPath();
3590             he.hdfsRegionDirModTime = regionDir.getModificationTime();
3591             Path regioninfoFile = new Path(he.hdfsRegionDir, HRegionFileSystem.REGION_INFO_FILE);
3592             he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
3593             // we add to orphan list when we attempt to read .regioninfo
3594 
3595             // Set a flag if this region contains only edits
3596             // This is special case if a region is left after split
3597             he.hdfsOnlyEdits = true;
3598             FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
3599             Path ePath = HLogUtil.getRegionDirRecoveredEditsDir(regionDir.getPath());
3600             for (FileStatus subDir : subDirs) {
3601               String sdName = subDir.getPath().getName();
3602               if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
3603                 he.hdfsOnlyEdits = false;
3604                 break;
3605               }
3606             }
3607             hbi.hdfsEntry = he;
3608           }
3609         }
3610       } catch (IOException e) {
3611         // unable to connect to the region server.
3612         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
3613             + tableDir.getPath().getName()
3614             + " Unable to fetch region information. " + e);
3615         throw e;
3616       }
3617       return null;
3618     }
3619   }
3620 
3621   /**
3622    * Contact hdfs and get all information about specified table directory into
3623    * regioninfo list.
3624    */
3625   static class WorkItemHdfsRegionInfo implements Callable<Void> {
3626     private HbckInfo hbi;
3627     private HBaseFsck hbck;
3628     private ErrorReporter errors;
3629 
3630     WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
3631       this.hbi = hbi;
3632       this.hbck = hbck;
3633       this.errors = errors;
3634     }
3635 
3636     @Override
3637     public synchronized Void call() throws IOException {
3638       // only load entries that haven't been loaded yet.
3639       if (hbi.getHdfsHRI() == null) {
3640         try {
3641           hbck.loadHdfsRegioninfo(hbi);
3642         } catch (IOException ioe) {
3643           String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
3644               + hbi.getTableName() + " in hdfs dir "
3645               + hbi.getHdfsRegionDir()
3646               + "!  It may be an invalid format or version file.  Treating as "
3647               + "an orphaned regiondir.";
3648           errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
3649           try {
3650             hbck.debugLsr(hbi.getHdfsRegionDir());
3651           } catch (IOException ioe2) {
3652             LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
3653             throw ioe2;
3654           }
3655           hbck.orphanHdfsDirs.add(hbi);
3656           throw ioe;
3657         }
3658       }
3659       return null;
3660     }
3661   };
3662 
3663   /**
3664    * Display the full report from fsck. This displays all live and dead region
3665    * servers, and all known regions.
3666    */
3667   public static void setDisplayFullReport() {
3668     details = true;
3669   }
3670 
3671   /**
3672    * Set summary mode.
3673    * Print only summary of the tables and status (OK or INCONSISTENT)
3674    */
3675   void setSummary() {
3676     summary = true;
3677   }
3678 
3679   /**
3680    * Set hbase:meta check mode.
3681    * Print only info about hbase:meta table deployment/state
3682    */
3683   void setCheckMetaOnly() {
3684     checkMetaOnly = true;
3685   }
3686 
3687   /**
3688    * Set region boundaries check mode.
3689    */
3690   void setRegionBoundariesCheck() {
3691     checkRegionBoundaries = true;
3692   }
3693 
3694   /**
3695    * Set table locks fix mode.
3696    * Delete table locks held for a long time
3697    */
3698   public void setFixTableLocks(boolean shouldFix) {
3699     fixTableLocks = shouldFix;
3700   }
3701 
3702   /**
3703    * Check if we should rerun fsck again. This checks if we've tried to
3704    * fix something and we should rerun fsck tool again.
3705    * Display the full report from fsck. This displays all live and dead
3706    * region servers, and all known regions.
3707    */
3708   void setShouldRerun() {
3709     rerun = true;
3710   }
3711 
3712   boolean shouldRerun() {
3713     return rerun;
3714   }
3715 
3716   /**
3717    * Fix inconsistencies found by fsck. This should try to fix errors (if any)
3718    * found by fsck utility.
3719    */
3720   public void setFixAssignments(boolean shouldFix) {
3721     fixAssignments = shouldFix;
3722   }
3723 
3724   boolean shouldFixAssignments() {
3725     return fixAssignments;
3726   }
3727 
3728   public void setFixMeta(boolean shouldFix) {
3729     fixMeta = shouldFix;
3730   }
3731 
3732   boolean shouldFixMeta() {
3733     return fixMeta;
3734   }
3735 
3736   public void setFixEmptyMetaCells(boolean shouldFix) {
3737     fixEmptyMetaCells = shouldFix;
3738   }
3739 
3740   boolean shouldFixEmptyMetaCells() {
3741     return fixEmptyMetaCells;
3742   }
3743 
3744   public void setCheckHdfs(boolean checking) {
3745     checkHdfs = checking;
3746   }
3747 
3748   boolean shouldCheckHdfs() {
3749     return checkHdfs;
3750   }
3751 
3752   public void setFixHdfsHoles(boolean shouldFix) {
3753     fixHdfsHoles = shouldFix;
3754   }
3755 
3756   boolean shouldFixHdfsHoles() {
3757     return fixHdfsHoles;
3758   }
3759 
3760   public void setFixTableOrphans(boolean shouldFix) {
3761     fixTableOrphans = shouldFix;
3762   }
3763 
3764   boolean shouldFixTableOrphans() {
3765     return fixTableOrphans;
3766   }
3767 
3768   public void setFixHdfsOverlaps(boolean shouldFix) {
3769     fixHdfsOverlaps = shouldFix;
3770   }
3771 
3772   boolean shouldFixHdfsOverlaps() {
3773     return fixHdfsOverlaps;
3774   }
3775 
3776   public void setFixHdfsOrphans(boolean shouldFix) {
3777     fixHdfsOrphans = shouldFix;
3778   }
3779 
3780   boolean shouldFixHdfsOrphans() {
3781     return fixHdfsOrphans;
3782   }
3783 
3784   public void setFixVersionFile(boolean shouldFix) {
3785     fixVersionFile = shouldFix;
3786   }
3787 
3788   public boolean shouldFixVersionFile() {
3789     return fixVersionFile;
3790   }
3791 
3792   public void setSidelineBigOverlaps(boolean sbo) {
3793     this.sidelineBigOverlaps = sbo;
3794   }
3795 
3796   public boolean shouldSidelineBigOverlaps() {
3797     return sidelineBigOverlaps;
3798   }
3799 
3800   public void setFixSplitParents(boolean shouldFix) {
3801     fixSplitParents = shouldFix;
3802   }
3803 
3804   boolean shouldFixSplitParents() {
3805     return fixSplitParents;
3806   }
3807 
3808   public void setFixReferenceFiles(boolean shouldFix) {
3809     fixReferenceFiles = shouldFix;
3810   }
3811 
3812   boolean shouldFixReferenceFiles() {
3813     return fixReferenceFiles;
3814   }
3815 
3816   public boolean shouldIgnorePreCheckPermission() {
3817     return ignorePreCheckPermission;
3818   }
3819 
3820   public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
3821     this.ignorePreCheckPermission = ignorePreCheckPermission;
3822   }
3823 
3824   /**
3825    * @param mm maximum number of regions to merge into a single region.
3826    */
3827   public void setMaxMerge(int mm) {
3828     this.maxMerge = mm;
3829   }
3830 
3831   public int getMaxMerge() {
3832     return maxMerge;
3833   }
3834 
3835   public void setMaxOverlapsToSideline(int mo) {
3836     this.maxOverlapsToSideline = mo;
3837   }
3838 
3839   public int getMaxOverlapsToSideline() {
3840     return maxOverlapsToSideline;
3841   }
3842 
3843   /**
3844    * Only check/fix tables specified by the list,
3845    * Empty list means all tables are included.
3846    */
3847   boolean isTableIncluded(TableName table) {
3848     return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
3849   }
3850 
3851   public void includeTable(TableName table) {
3852     tablesIncluded.add(table);
3853   }
3854 
3855   Set<TableName> getIncludedTables() {
3856     return new HashSet<TableName>(tablesIncluded);
3857   }
3858 
3859   /**
3860    * We are interested in only those tables that have not changed their state in
3861    * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
3862    * @param seconds - the time in seconds
3863    */
3864   public void setTimeLag(long seconds) {
3865     timelag = seconds * 1000; // convert to milliseconds
3866   }
3867 
3868   /**
3869    *
3870    * @param sidelineDir - HDFS path to sideline data
3871    */
3872   public void setSidelineDir(String sidelineDir) {
3873     this.sidelineDir = new Path(sidelineDir);
3874   }
3875 
3876   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
3877     return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
3878   }
3879 
3880   public HFileCorruptionChecker getHFilecorruptionChecker() {
3881     return hfcc;
3882   }
3883 
3884   public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
3885     this.hfcc = hfcc;
3886   }
3887 
3888   public void setRetCode(int code) {
3889     this.retcode = code;
3890   }
3891 
3892   public int getRetCode() {
3893     return retcode;
3894   }
3895 
3896   protected HBaseFsck printUsageAndExit() {
3897     StringWriter sw = new StringWriter(2048);
3898     PrintWriter out = new PrintWriter(sw);
3899     out.println("Usage: fsck [opts] {only tables}");
3900     out.println(" where [opts] are:");
3901     out.println("   -help Display help options (this)");
3902     out.println("   -details Display full report of all regions.");
3903     out.println("   -timelag <timeInSeconds>  Process only regions that " +
3904                        " have not experienced any metadata updates in the last " +
3905                        " <timeInSeconds> seconds.");
3906     out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
3907         " before checking if the fix worked if run with -fix");
3908     out.println("   -summary Print only summary of the tables and status.");
3909     out.println("   -metaonly Only check the state of the hbase:meta table.");
3910     out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
3911     out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
3912 
3913     out.println("");
3914     out.println("  Metadata Repair options: (expert features, use with caution!)");
3915     out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
3916     out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
3917     out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
3918     out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
3919         + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
3920     out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
3921     out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
3922     out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
3923     out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
3924     out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
3925     out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
3926     out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
3927     out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
3928     out.println("   -fixSplitParents  Try to force offline split parents to be online.");
3929     out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
3930     out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
3931     out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
3932         + " (empty REGIONINFO_QUALIFIER rows)");
3933 
3934     out.println("");
3935     out.println("  Datafile Repair options: (expert features, use with caution!)");
3936     out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
3937     out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
3938 
3939     out.println("");
3940     out.println("  Metadata Repair shortcuts");
3941     out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
3942         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles -fixTableLocks");
3943     out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
3944 
3945     out.println("");
3946     out.println("  Table lock options");
3947     out.println("   -fixTableLocks    Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
3948 
3949     out.flush();
3950     errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
3951 
3952     setRetCode(-2);
3953     return this;
3954   }
3955 
3956   /**
3957    * Main program
3958    *
3959    * @param args
3960    * @throws Exception
3961    */
3962   public static void main(String[] args) throws Exception {
3963     // create a fsck object
3964     Configuration conf = HBaseConfiguration.create();
3965     Path hbasedir = FSUtils.getRootDir(conf);
3966     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
3967     FSUtils.setFsDefault(conf, new Path(defaultFs));
3968 
3969     int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
3970     System.exit(ret);
3971   }
3972 
3973   /**
3974    * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
3975    */
3976   static class HBaseFsckTool extends Configured implements Tool {
3977     HBaseFsckTool(Configuration conf) { super(conf); }
3978     @Override
3979     public int run(String[] args) throws Exception {
3980       HBaseFsck hbck = new HBaseFsck(getConf());
3981       hbck.exec(hbck.executor, args);
3982       return hbck.getRetCode();
3983     }
3984   };
3985 
3986 
3987   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
3988     ServiceException, InterruptedException {
3989     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
3990 
3991     boolean checkCorruptHFiles = false;
3992     boolean sidelineCorruptHFiles = false;
3993 
3994     // Process command-line args.
3995     for (int i = 0; i < args.length; i++) {
3996       String cmd = args[i];
3997       if (cmd.equals("-help") || cmd.equals("-h")) {
3998         return printUsageAndExit();
3999       } else if (cmd.equals("-details")) {
4000         setDisplayFullReport();
4001       } else if (cmd.equals("-timelag")) {
4002         if (i == args.length - 1) {
4003           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4004           return printUsageAndExit();
4005         }
4006         try {
4007           long timelag = Long.parseLong(args[i+1]);
4008           setTimeLag(timelag);
4009         } catch (NumberFormatException e) {
4010           errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4011           return printUsageAndExit();
4012         }
4013         i++;
4014       } else if (cmd.equals("-sleepBeforeRerun")) {
4015         if (i == args.length - 1) {
4016           errors.reportError(ERROR_CODE.WRONG_USAGE,
4017             "HBaseFsck: -sleepBeforeRerun needs a value.");
4018           return printUsageAndExit();
4019         }
4020         try {
4021           sleepBeforeRerun = Long.parseLong(args[i+1]);
4022         } catch (NumberFormatException e) {
4023           errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4024           return printUsageAndExit();
4025         }
4026         i++;
4027       } else if (cmd.equals("-sidelineDir")) {
4028         if (i == args.length - 1) {
4029           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4030           return printUsageAndExit();
4031         }
4032         i++;
4033         setSidelineDir(args[i]);
4034       } else if (cmd.equals("-fix")) {
4035         errors.reportError(ERROR_CODE.WRONG_USAGE,
4036           "This option is deprecated, please use  -fixAssignments instead.");
4037         setFixAssignments(true);
4038       } else if (cmd.equals("-fixAssignments")) {
4039         setFixAssignments(true);
4040       } else if (cmd.equals("-fixMeta")) {
4041         setFixMeta(true);
4042       } else if (cmd.equals("-noHdfsChecking")) {
4043         setCheckHdfs(false);
4044       } else if (cmd.equals("-fixHdfsHoles")) {
4045         setFixHdfsHoles(true);
4046       } else if (cmd.equals("-fixHdfsOrphans")) {
4047         setFixHdfsOrphans(true);
4048       } else if (cmd.equals("-fixTableOrphans")) {
4049         setFixTableOrphans(true);
4050       } else if (cmd.equals("-fixHdfsOverlaps")) {
4051         setFixHdfsOverlaps(true);
4052       } else if (cmd.equals("-fixVersionFile")) {
4053         setFixVersionFile(true);
4054       } else if (cmd.equals("-sidelineBigOverlaps")) {
4055         setSidelineBigOverlaps(true);
4056       } else if (cmd.equals("-fixSplitParents")) {
4057         setFixSplitParents(true);
4058       } else if (cmd.equals("-ignorePreCheckPermission")) {
4059         setIgnorePreCheckPermission(true);
4060       } else if (cmd.equals("-checkCorruptHFiles")) {
4061         checkCorruptHFiles = true;
4062       } else if (cmd.equals("-sidelineCorruptHFiles")) {
4063         sidelineCorruptHFiles = true;
4064       } else if (cmd.equals("-fixReferenceFiles")) {
4065         setFixReferenceFiles(true);
4066       } else if (cmd.equals("-fixEmptyMetaCells")) {
4067         setFixEmptyMetaCells(true);
4068       } else if (cmd.equals("-repair")) {
4069         // this attempts to merge overlapping hdfs regions, needs testing
4070         // under load
4071         setFixHdfsHoles(true);
4072         setFixHdfsOrphans(true);
4073         setFixMeta(true);
4074         setFixAssignments(true);
4075         setFixHdfsOverlaps(true);
4076         setFixVersionFile(true);
4077         setSidelineBigOverlaps(true);
4078         setFixSplitParents(false);
4079         setCheckHdfs(true);
4080         setFixReferenceFiles(true);
4081         setFixTableLocks(true);
4082       } else if (cmd.equals("-repairHoles")) {
4083         // this will make all missing hdfs regions available but may lose data
4084         setFixHdfsHoles(true);
4085         setFixHdfsOrphans(false);
4086         setFixMeta(true);
4087         setFixAssignments(true);
4088         setFixHdfsOverlaps(false);
4089         setSidelineBigOverlaps(false);
4090         setFixSplitParents(false);
4091         setCheckHdfs(true);
4092       } else if (cmd.equals("-maxOverlapsToSideline")) {
4093         if (i == args.length - 1) {
4094           errors.reportError(ERROR_CODE.WRONG_USAGE,
4095             "-maxOverlapsToSideline needs a numeric value argument.");
4096           return printUsageAndExit();
4097         }
4098         try {
4099           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4100           setMaxOverlapsToSideline(maxOverlapsToSideline);
4101         } catch (NumberFormatException e) {
4102           errors.reportError(ERROR_CODE.WRONG_USAGE,
4103             "-maxOverlapsToSideline needs a numeric value argument.");
4104           return printUsageAndExit();
4105         }
4106         i++;
4107       } else if (cmd.equals("-maxMerge")) {
4108         if (i == args.length - 1) {
4109           errors.reportError(ERROR_CODE.WRONG_USAGE,
4110             "-maxMerge needs a numeric value argument.");
4111           return printUsageAndExit();
4112         }
4113         try {
4114           int maxMerge = Integer.parseInt(args[i+1]);
4115           setMaxMerge(maxMerge);
4116         } catch (NumberFormatException e) {
4117           errors.reportError(ERROR_CODE.WRONG_USAGE,
4118             "-maxMerge needs a numeric value argument.");
4119           return printUsageAndExit();
4120         }
4121         i++;
4122       } else if (cmd.equals("-summary")) {
4123         setSummary();
4124       } else if (cmd.equals("-metaonly")) {
4125         setCheckMetaOnly();
4126       } else if (cmd.equals("-boundaries")) {
4127         setRegionBoundariesCheck();
4128       } else if (cmd.equals("-fixTableLocks")) {
4129         setFixTableLocks(true);
4130       } else if (cmd.startsWith("-")) {
4131         errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4132         return printUsageAndExit();
4133       } else {
4134         includeTable(TableName.valueOf(cmd));
4135         errors.print("Allow checking/fixes for table: " + cmd);
4136       }
4137     }
4138 
4139     // pre-check current user has FS write permission or not
4140     try {
4141       preCheckPermission();
4142     } catch (AccessDeniedException ace) {
4143       Runtime.getRuntime().exit(-1);
4144     } catch (IOException ioe) {
4145       Runtime.getRuntime().exit(-1);
4146     }
4147 
4148     // do the real work of hbck
4149     connect();
4150 
4151     try {
4152       // if corrupt file mode is on, first fix them since they may be opened later
4153       if (checkCorruptHFiles || sidelineCorruptHFiles) {
4154         LOG.info("Checking all hfiles for corruption");
4155         HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4156         setHFileCorruptionChecker(hfcc); // so we can get result
4157         Collection<TableName> tables = getIncludedTables();
4158         Collection<Path> tableDirs = new ArrayList<Path>();
4159         Path rootdir = FSUtils.getRootDir(getConf());
4160         if (tables.size() > 0) {
4161           for (TableName t : tables) {
4162             tableDirs.add(FSUtils.getTableDir(rootdir, t));
4163           }
4164         } else {
4165           tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4166         }
4167         hfcc.checkTables(tableDirs);
4168         hfcc.report(errors);
4169       }
4170 
4171       // check and fix table integrity, region consistency.
4172       int code = onlineHbck();
4173       setRetCode(code);
4174       // If we have changed the HBase state it is better to run hbck again
4175       // to see if we haven't broken something else in the process.
4176       // We run it only once more because otherwise we can easily fall into
4177       // an infinite loop.
4178       if (shouldRerun()) {
4179         try {
4180           LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4181           Thread.sleep(sleepBeforeRerun);
4182         } catch (InterruptedException ie) {
4183           LOG.warn("Interrupted while sleeping");
4184           return this;
4185         }
4186         // Just report
4187         setFixAssignments(false);
4188         setFixMeta(false);
4189         setFixHdfsHoles(false);
4190         setFixHdfsOverlaps(false);
4191         setFixVersionFile(false);
4192         setFixTableOrphans(false);
4193         errors.resetErrors();
4194         code = onlineHbck();
4195         setRetCode(code);
4196       }
4197     } finally {
4198       IOUtils.cleanup(null, connection, meta, admin);
4199     }
4200     return this;
4201   }
4202 
4203   /**
4204    * ls -r for debugging purposes
4205    */
4206   void debugLsr(Path p) throws IOException {
4207     debugLsr(getConf(), p, errors);
4208   }
4209 
4210   /**
4211    * ls -r for debugging purposes
4212    */
4213   public static void debugLsr(Configuration conf,
4214       Path p) throws IOException {
4215     debugLsr(conf, p, new PrintingErrorReporter());
4216   }
4217 
4218   /**
4219    * ls -r for debugging purposes
4220    */
4221   public static void debugLsr(Configuration conf,
4222       Path p, ErrorReporter errors) throws IOException {
4223     if (!LOG.isDebugEnabled() || p == null) {
4224       return;
4225     }
4226     FileSystem fs = p.getFileSystem(conf);
4227 
4228     if (!fs.exists(p)) {
4229       // nothing
4230       return;
4231     }
4232     errors.print(p.toString());
4233 
4234     if (fs.isFile(p)) {
4235       return;
4236     }
4237 
4238     if (fs.getFileStatus(p).isDirectory()) {
4239       FileStatus[] fss= fs.listStatus(p);
4240       for (FileStatus status : fss) {
4241         debugLsr(conf, status.getPath(), errors);
4242       }
4243     }
4244   }
4245 }