View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.FileNotFoundException;
21  import java.io.IOException;
22  import java.io.PrintWriter;
23  import java.io.StringWriter;
24  import java.net.URI;
25  import java.util.ArrayList;
26  import java.util.Arrays;
27  import java.util.Collection;
28  import java.util.Collections;
29  import java.util.Comparator;
30  import java.util.HashMap;
31  import java.util.HashSet;
32  import java.util.Iterator;
33  import java.util.List;
34  import java.util.Map;
35  import java.util.Map.Entry;
36  import java.util.Set;
37  import java.util.SortedMap;
38  import java.util.SortedSet;
39  import java.util.TreeMap;
40  import java.util.TreeSet;
41  import java.util.concurrent.Callable;
42  import java.util.concurrent.ConcurrentSkipListMap;
43  import java.util.concurrent.ExecutionException;
44  import java.util.concurrent.ExecutorService;
45  import java.util.concurrent.Future;
46  import java.util.concurrent.ScheduledThreadPoolExecutor;
47  import java.util.concurrent.atomic.AtomicInteger;
48  
49  import org.apache.commons.logging.Log;
50  import org.apache.commons.logging.LogFactory;
51  import org.apache.hadoop.classification.InterfaceAudience;
52  import org.apache.hadoop.classification.InterfaceStability;
53  import org.apache.hadoop.conf.Configuration;
54  import org.apache.hadoop.conf.Configured;
55  import org.apache.hadoop.fs.FileStatus;
56  import org.apache.hadoop.fs.FileSystem;
57  import org.apache.hadoop.fs.Path;
58  import org.apache.hadoop.fs.permission.FsAction;
59  import org.apache.hadoop.hbase.Abortable;
60  import org.apache.hadoop.hbase.Cell;
61  import org.apache.hadoop.hbase.ClusterStatus;
62  import org.apache.hadoop.hbase.HBaseConfiguration;
63  import org.apache.hadoop.hbase.HColumnDescriptor;
64  import org.apache.hadoop.hbase.HConstants;
65  import org.apache.hadoop.hbase.HRegionInfo;
66  import org.apache.hadoop.hbase.HRegionLocation;
67  import org.apache.hadoop.hbase.HTableDescriptor;
68  import org.apache.hadoop.hbase.KeyValue;
69  import org.apache.hadoop.hbase.MasterNotRunningException;
70  import org.apache.hadoop.hbase.RegionLocations;
71  import org.apache.hadoop.hbase.ServerName;
72  import org.apache.hadoop.hbase.TableDescriptor;
73  import org.apache.hadoop.hbase.TableName;
74  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
75  import org.apache.hadoop.hbase.MetaTableAccessor;
76  import org.apache.hadoop.hbase.client.Admin;
77  import org.apache.hadoop.hbase.client.Delete;
78  import org.apache.hadoop.hbase.client.Get;
79  import org.apache.hadoop.hbase.client.HBaseAdmin;
80  import org.apache.hadoop.hbase.client.HConnectable;
81  import org.apache.hadoop.hbase.client.HConnection;
82  import org.apache.hadoop.hbase.client.HConnectionManager;
83  import org.apache.hadoop.hbase.client.HTable;
84  import org.apache.hadoop.hbase.client.MetaScanner;
85  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
86  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
87  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
88  import org.apache.hadoop.hbase.client.Put;
89  import org.apache.hadoop.hbase.client.Result;
90  import org.apache.hadoop.hbase.client.RowMutations;
91  import org.apache.hadoop.hbase.client.Table;
92  import org.apache.hadoop.hbase.client.TableState;
93  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
94  import org.apache.hadoop.hbase.io.hfile.HFile;
95  import org.apache.hadoop.hbase.master.MasterFileSystem;
96  import org.apache.hadoop.hbase.master.RegionState;
97  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
98  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
99  import org.apache.hadoop.hbase.regionserver.HRegion;
100 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
101 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
102 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
103 import org.apache.hadoop.hbase.security.UserProvider;
104 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
105 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
106 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
107 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
108 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
109 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
110 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
111 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
112 import org.apache.hadoop.hbase.security.AccessDeniedException;
113 import org.apache.hadoop.io.IOUtils;
114 import org.apache.hadoop.security.UserGroupInformation;
115 import org.apache.hadoop.util.ReflectionUtils;
116 import org.apache.hadoop.util.Tool;
117 import org.apache.hadoop.util.ToolRunner;
118 import org.apache.zookeeper.KeeperException;
119 
120 import com.google.common.base.Joiner;
121 import com.google.common.base.Preconditions;
122 import com.google.common.collect.Lists;
123 import com.google.common.collect.Multimap;
124 import com.google.common.collect.TreeMultimap;
125 import com.google.protobuf.ServiceException;
126 
127 /**
128  * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
129  * table integrity problems in a corrupted HBase.
130  * <p>
131  * Region consistency checks verify that hbase:meta, region deployment on region
132  * servers and the state of data in HDFS (.regioninfo files) all are in
133  * accordance.
134  * <p>
135  * Table integrity checks verify that all possible row keys resolve to exactly
136  * one region of a table.  This means there are no individual degenerate
137  * or backwards regions; no holes between regions; and that there are no
138  * overlapping regions.
139  * <p>
140  * The general repair strategy works in two phases:
141  * <ol>
142  * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
143  * <li> Repair Region Consistency with hbase:meta and assignments
144  * </ol>
145  * <p>
146  * For table integrity repairs, the tables' region directories are scanned
147  * for .regioninfo files.  Each table's integrity is then verified.  If there
148  * are any orphan regions (regions with no .regioninfo files) or holes, new
149  * regions are fabricated.  Backwards regions are sidelined as well as empty
150  * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
151  * a new region is created and all data is merged into the new region.
152  * <p>
153  * Table integrity repairs deal solely with HDFS and could potentially be done
154  * offline -- the hbase region servers or master do not need to be running.
155  * This phase can eventually be used to completely reconstruct the hbase:meta table in
156  * an offline fashion.
157  * <p>
158  * Region consistency requires three conditions -- 1) valid .regioninfo file
159  * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
160  * and 3) a region is deployed only at the regionserver that was assigned to
161  * with proper state in the master.
162  * <p>
163  * Region consistency repairs require hbase to be online so that hbck can
164  * contact the HBase master and region servers.  The hbck#connect() method must
165  * first be called successfully.  Much of the region consistency information
166  * is transient and less risky to repair.
167  * <p>
168  * If hbck is run from the command line, there are a handful of arguments that
169  * can be used to limit the kinds of repairs hbck will do.  See the code in
170  * {@link #printUsageAndExit()} for more details.
171  */
172 @InterfaceAudience.Public
173 @InterfaceStability.Evolving
174 public class HBaseFsck extends Configured {
175   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
176   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
177   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
178   private static boolean rsSupportsOffline = true;
179   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
180   private static final int DEFAULT_MAX_MERGE = 5;
181   private static final String TO_BE_LOADED = "to_be_loaded";
182 
183   /**********************
184    * Internal resources
185    **********************/
186   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
187   private ClusterStatus status;
188   private HConnection connection;
189   private Admin admin;
190   private Table meta;
191   // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
192   protected ExecutorService executor;
193   private long startMillis = System.currentTimeMillis();
194   private HFileCorruptionChecker hfcc;
195   private int retcode = 0;
196 
197   /***********
198    * Options
199    ***********/
200   private static boolean details = false; // do we display the full report
201   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
202   private boolean fixAssignments = false; // fix assignment errors?
203   private boolean fixMeta = false; // fix meta errors?
204   private boolean checkHdfs = true; // load and check fs consistency?
205   private boolean fixHdfsHoles = false; // fix fs holes?
206   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
207   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
208   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
209   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
210   private boolean fixSplitParents = false; // fix lingering split parents
211   private boolean fixReferenceFiles = false; // fix lingering reference store file
212   private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
213   private boolean fixTableLocks = false; // fix table locks which are expired
214 
215   // limit checking/fixes to listed tables, if empty attempt to check/fix all
216   // hbase:meta are always checked
217   private Set<TableName> tablesIncluded = new HashSet<TableName>();
218   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
219   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
220   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
221   private Path sidelineDir = null;
222 
223   private boolean rerun = false; // if we tried to fix something, rerun hbck
224   private static boolean summary = false; // if we want to print less output
225   private boolean checkMetaOnly = false;
226   private boolean checkRegionBoundaries = false;
227   private boolean ignorePreCheckPermission = false; // if pre-check permission
228 
229   /*********
230    * State
231    *********/
232   final private ErrorReporter errors;
233   int fixes = 0;
234 
235   /**
236    * This map contains the state of all hbck items.  It maps from encoded region
237    * name to HbckInfo structure.  The information contained in HbckInfo is used
238    * to detect and correct consistency (hdfs/meta/deployment) problems.
239    */
240   private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
241   private TreeSet<TableName> disabledTables =
242     new TreeSet<TableName>();
243   // Empty regioninfo qualifiers in hbase:meta
244   private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
245 
246   /**
247    * This map from Tablename -> TableInfo contains the structures necessary to
248    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
249    * to prevent dupes.
250    *
251    * If tablesIncluded is empty, this map contains all tables.
252    * Otherwise, it contains only meta tables and tables in tablesIncluded,
253    * unless checkMetaOnly is specified, in which case, it contains only
254    * the meta table
255    */
256   private SortedMap<TableName, TableInfo> tablesInfo =
257       new ConcurrentSkipListMap<TableName, TableInfo>();
258 
259   /**
260    * When initially looking at HDFS, we attempt to find any orphaned data.
261    */
262   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
263 
264   private Map<TableName, Set<String>> orphanTableDirs =
265       new HashMap<TableName, Set<String>>();
266 
267   /**
268    * Constructor
269    *
270    * @param conf Configuration object
271    * @throws MasterNotRunningException if the master is not running
272    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
273    */
274   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
275       ZooKeeperConnectionException, IOException, ClassNotFoundException {
276     super(conf);
277     // make a copy, just to be sure we're not overriding someone else's config
278     setConf(HBaseConfiguration.create(getConf()));
279     // disable blockcache for tool invocation, see HBASE-10500
280     getConf().setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0);
281     errors = getErrorReporter(conf);
282 
283     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
284     executor = new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
285   }
286 
287   /**
288    * Constructor
289    *
290    * @param conf
291    *          Configuration object
292    * @throws MasterNotRunningException
293    *           if the master is not running
294    * @throws ZooKeeperConnectionException
295    *           if unable to connect to ZooKeeper
296    */
297   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
298       ZooKeeperConnectionException, IOException, ClassNotFoundException {
299     super(conf);
300     errors = getErrorReporter(getConf());
301     this.executor = exec;
302   }
303 
304   /**
305    * To repair region consistency, one must call connect() in order to repair
306    * online state.
307    */
308   public void connect() throws IOException {
309     connection = HConnectionManager.createConnection(getConf());
310     admin = new HBaseAdmin(connection);
311     meta = new HTable(TableName.META_TABLE_NAME, connection);
312     status = admin.getClusterStatus();
313   }
314 
315   /**
316    * Get deployed regions according to the region servers.
317    */
318   private void loadDeployedRegions() throws IOException, InterruptedException {
319     // From the master, get a list of all known live region servers
320     Collection<ServerName> regionServers = status.getServers();
321     errors.print("Number of live region servers: " + regionServers.size());
322     if (details) {
323       for (ServerName rsinfo: regionServers) {
324         errors.print("  " + rsinfo.getServerName());
325       }
326     }
327 
328     // From the master, get a list of all dead region servers
329     Collection<ServerName> deadRegionServers = status.getDeadServerNames();
330     errors.print("Number of dead region servers: " + deadRegionServers.size());
331     if (details) {
332       for (ServerName name: deadRegionServers) {
333         errors.print("  " + name);
334       }
335     }
336 
337     // Print the current master name and state
338     errors.print("Master: " + status.getMaster());
339 
340     // Print the list of all backup masters
341     Collection<ServerName> backupMasters = status.getBackupMasters();
342     errors.print("Number of backup masters: " + backupMasters.size());
343     if (details) {
344       for (ServerName name: backupMasters) {
345         errors.print("  " + name);
346       }
347     }
348 
349     errors.print("Average load: " + status.getAverageLoad());
350     errors.print("Number of requests: " + status.getRequestsCount());
351     errors.print("Number of regions: " + status.getRegionsCount());
352 
353     Map<String, RegionState> rits = status.getRegionsInTransition();
354     errors.print("Number of regions in transition: " + rits.size());
355     if (details) {
356       for (RegionState state: rits.values()) {
357         errors.print("  " + state.toDescriptiveString());
358       }
359     }
360 
361     // Determine what's deployed
362     processRegionServers(regionServers);
363   }
364 
365   /**
366    * Clear the current state of hbck.
367    */
368   private void clearState() {
369     // Make sure regionInfo is empty before starting
370     fixes = 0;
371     regionInfoMap.clear();
372     emptyRegionInfoQualifiers.clear();
373     disabledTables.clear();
374     errors.clear();
375     tablesInfo.clear();
376     orphanHdfsDirs.clear();
377   }
378 
379   /**
380    * This repair method analyzes hbase data in hdfs and repairs it to satisfy
381    * the table integrity rules.  HBase doesn't need to be online for this
382    * operation to work.
383    */
384   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
385     // Initial pass to fix orphans.
386     if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
387         || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
388       LOG.info("Loading regioninfos HDFS");
389       // if nothing is happening this should always complete in two iterations.
390       int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
391       int curIter = 0;
392       do {
393         clearState(); // clears hbck state and reset fixes to 0 and.
394         // repair what's on HDFS
395         restoreHdfsIntegrity();
396         curIter++;// limit the number of iterations.
397       } while (fixes > 0 && curIter <= maxIterations);
398 
399       // Repairs should be done in the first iteration and verification in the second.
400       // If there are more than 2 passes, something funny has happened.
401       if (curIter > 2) {
402         if (curIter == maxIterations) {
403           LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
404               + "Tables integrity may not be fully repaired!");
405         } else {
406           LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
407         }
408       }
409     }
410   }
411 
412   /**
413    * This repair method requires the cluster to be online since it contacts
414    * region servers and the masters.  It makes each region's state in HDFS, in
415    * hbase:meta, and deployments consistent.
416    *
417    * @return If > 0 , number of errors detected, if < 0 there was an unrecoverable
418    * error.  If 0, we have a clean hbase.
419    */
420   public int onlineConsistencyRepair() throws IOException, KeeperException,
421     InterruptedException {
422     clearState();
423 
424     // get regions according to what is online on each RegionServer
425     loadDeployedRegions();
426     // check whether hbase:meta is deployed and online
427     recordMetaRegion();
428     // Check if hbase:meta is found only once and in the right place
429     if (!checkMetaRegion()) {
430       String errorMsg = "hbase:meta table is not consistent. ";
431       if (shouldFixAssignments()) {
432         errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
433       } else {
434         errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
435       }
436       errors.reportError(errorMsg + " Exiting...");
437       return -2;
438     }
439     // Not going with further consistency check for tables when hbase:meta itself is not consistent.
440     LOG.info("Loading regionsinfo from the hbase:meta table");
441     boolean success = loadMetaEntries();
442     if (!success) return -1;
443 
444     // Empty cells in hbase:meta?
445     reportEmptyMetaCells();
446 
447     // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
448     if (shouldFixEmptyMetaCells()) {
449       fixEmptyMetaCells();
450     }
451 
452     // get a list of all tables that have not changed recently.
453     if (!checkMetaOnly) {
454       reportTablesInFlux();
455     }
456 
457     // load regiondirs and regioninfos from HDFS
458     if (shouldCheckHdfs()) {
459       loadHdfsRegionDirs();
460       loadHdfsRegionInfos();
461     }
462 
463     // Get disabled tables from ZooKeeper
464     loadDisabledTables();
465 
466     // fix the orphan tables
467     fixOrphanTables();
468 
469     // Check and fix consistency
470     checkAndFixConsistency();
471 
472     // Check integrity (does not fix)
473     checkIntegrity();
474     return errors.getErrorList().size();
475   }
476 
477   /**
478    * Contacts the master and prints out cluster-wide information
479    * @return 0 on success, non-zero on failure
480    */
481   public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
482     // print hbase server version
483     errors.print("Version: " + status.getHBaseVersion());
484     offlineHdfsIntegrityRepair();
485 
486     // turn the balancer off
487     boolean oldBalancer = admin.setBalancerRunning(false, true);
488     try {
489       onlineConsistencyRepair();
490     }
491     finally {
492       admin.setBalancerRunning(oldBalancer, false);
493     }
494 
495     if (checkRegionBoundaries) {
496       checkRegionBoundaries();
497     }
498 
499     offlineReferenceFileRepair();
500 
501     checkAndFixTableLocks();
502 
503     // Print table summary
504     printTableSummary(tablesInfo);
505     return errors.summarize();
506   }
507 
508   public static byte[] keyOnly (byte[] b) {
509     if (b == null)
510       return b;
511     int rowlength = Bytes.toShort(b, 0);
512     byte[] result = new byte[rowlength];
513     System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
514     return result;
515   }
516 
517   private static class RegionBoundariesInformation {
518     public byte [] regionName;
519     public byte [] metaFirstKey;
520     public byte [] metaLastKey;
521     public byte [] storesFirstKey;
522     public byte [] storesLastKey;
523     @Override
524     public String toString () {
525       return "regionName=" + Bytes.toStringBinary(regionName) +
526              "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
527              "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
528              "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
529              "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
530     }
531   }
532 
533   public void checkRegionBoundaries() {
534     try {
535       ByteArrayComparator comparator = new ByteArrayComparator();
536       List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), false);
537       final RegionBoundariesInformation currentRegionBoundariesInformation =
538           new RegionBoundariesInformation();
539       Path hbaseRoot = FSUtils.getRootDir(getConf());
540       for (HRegionInfo regionInfo : regions) {
541         Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
542         currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
543         // For each region, get the start and stop key from the META and compare them to the
544         // same information from the Stores.
545         Path path = new Path(tableDir, regionInfo.getEncodedName());
546         FileSystem fs = path.getFileSystem(getConf());
547         FileStatus[] files = fs.listStatus(path);
548         // For all the column families in this region...
549         byte[] storeFirstKey = null;
550         byte[] storeLastKey = null;
551         for (FileStatus file : files) {
552           String fileName = file.getPath().toString();
553           fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
554           if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
555             FileStatus[] storeFiles = fs.listStatus(file.getPath());
556             // For all the stores in this column family.
557             for (FileStatus storeFile : storeFiles) {
558               HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
559                   getConf()), getConf());
560               if ((reader.getFirstKey() != null)
561                   && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
562                       reader.getFirstKey()) > 0))) {
563                 storeFirstKey = reader.getFirstKey();
564               }
565               if ((reader.getLastKey() != null)
566                   && ((storeLastKey == null) || (comparator.compare(storeLastKey,
567                       reader.getLastKey())) < 0)) {
568                 storeLastKey = reader.getLastKey();
569               }
570               reader.close();
571             }
572           }
573         }
574         currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
575         currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
576         currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
577         currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
578         if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
579           currentRegionBoundariesInformation.metaFirstKey = null;
580         if (currentRegionBoundariesInformation.metaLastKey.length == 0)
581           currentRegionBoundariesInformation.metaLastKey = null;
582 
583         // For a region to be correct, we need the META start key to be smaller or equal to the
584         // smallest start key from all the stores, and the start key from the next META entry to
585         // be bigger than the last key from all the current stores. First region start key is null;
586         // Last region end key is null; some regions can be empty and not have any store.
587 
588         boolean valid = true;
589         // Checking start key.
590         if ((currentRegionBoundariesInformation.storesFirstKey != null)
591             && (currentRegionBoundariesInformation.metaFirstKey != null)) {
592           valid = valid
593               && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
594                 currentRegionBoundariesInformation.metaFirstKey) >= 0;
595         }
596         // Checking stop key.
597         if ((currentRegionBoundariesInformation.storesLastKey != null)
598             && (currentRegionBoundariesInformation.metaLastKey != null)) {
599           valid = valid
600               && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
601                 currentRegionBoundariesInformation.metaLastKey) < 0;
602         }
603         if (!valid) {
604           errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
605             tablesInfo.get(regionInfo.getTable()));
606           LOG.warn("Region's boundaries not alligned between stores and META for:");
607           LOG.warn(currentRegionBoundariesInformation);
608         }
609       }
610     } catch (IOException e) {
611       LOG.error(e);
612     }
613   }
614 
615   /**
616    * Iterates through the list of all orphan/invalid regiondirs.
617    */
618   private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
619     for (HbckInfo hi : orphanHdfsDirs) {
620       LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
621       adoptHdfsOrphan(hi);
622     }
623   }
624 
625   /**
626    * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
627    * these orphans by creating a new region, and moving the column families,
628    * recovered edits, HLogs, into the new region dir.  We determine the region
629    * startkey and endkeys by looking at all of the hfiles inside the column
630    * families to identify the min and max keys. The resulting region will
631    * likely violate table integrity but will be dealt with by merging
632    * overlapping regions.
633    */
634   @SuppressWarnings("deprecation")
635   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
636     Path p = hi.getHdfsRegionDir();
637     FileSystem fs = p.getFileSystem(getConf());
638     FileStatus[] dirs = fs.listStatus(p);
639     if (dirs == null) {
640       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
641           p + ". This dir could probably be deleted.");
642       return ;
643     }
644 
645     TableName tableName = hi.getTableName();
646     TableInfo tableInfo = tablesInfo.get(tableName);
647     Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
648     HTableDescriptor template = tableInfo.getHTD();
649 
650     // find min and max key values
651     Pair<byte[],byte[]> orphanRegionRange = null;
652     for (FileStatus cf : dirs) {
653       String cfName= cf.getPath().getName();
654       // TODO Figure out what the special dirs are
655       if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
656 
657       FileStatus[] hfiles = fs.listStatus(cf.getPath());
658       for (FileStatus hfile : hfiles) {
659         byte[] start, end;
660         HFile.Reader hf = null;
661         try {
662           CacheConfig cacheConf = new CacheConfig(getConf());
663           hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
664           hf.loadFileInfo();
665           KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
666           start = startKv.getRow();
667           KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
668           end = endKv.getRow();
669         } catch (IOException ioe) {
670           LOG.warn("Problem reading orphan file " + hfile + ", skipping");
671           continue;
672         } catch (NullPointerException ioe) {
673           LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
674           continue;
675         } finally {
676           if (hf != null) {
677             hf.close();
678           }
679         }
680 
681         // expand the range to include the range of all hfiles
682         if (orphanRegionRange == null) {
683           // first range
684           orphanRegionRange = new Pair<byte[], byte[]>(start, end);
685         } else {
686           // TODO add test
687 
688           // expand range only if the hfile is wider.
689           if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
690             orphanRegionRange.setFirst(start);
691           }
692           if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
693             orphanRegionRange.setSecond(end);
694           }
695         }
696       }
697     }
698     if (orphanRegionRange == null) {
699       LOG.warn("No data in dir " + p + ", sidelining data");
700       fixes++;
701       sidelineRegionDir(fs, hi);
702       return;
703     }
704     LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
705         Bytes.toString(orphanRegionRange.getSecond()) + ")");
706 
707     // create new region on hdfs.  move data into place.
708     HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond());
709     LOG.info("Creating new region : " + hri);
710     HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
711     Path target = region.getRegionFileSystem().getRegionDir();
712 
713     // rename all the data to new region
714     mergeRegionDirs(target, hi);
715     fixes++;
716   }
717 
718   /**
719    * This method determines if there are table integrity errors in HDFS.  If
720    * there are errors and the appropriate "fix" options are enabled, the method
721    * will first correct orphan regions making them into legit regiondirs, and
722    * then reload to merge potentially overlapping regions.
723    *
724    * @return number of table integrity errors found
725    */
726   private int restoreHdfsIntegrity() throws IOException, InterruptedException {
727     // Determine what's on HDFS
728     LOG.info("Loading HBase regioninfo from HDFS...");
729     loadHdfsRegionDirs(); // populating regioninfo table.
730 
731     int errs = errors.getErrorList().size();
732     // First time just get suggestions.
733     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
734     checkHdfsIntegrity(false, false);
735 
736     if (errors.getErrorList().size() == errs) {
737       LOG.info("No integrity errors.  We are done with this phase. Glorious.");
738       return 0;
739     }
740 
741     if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
742       adoptHdfsOrphans(orphanHdfsDirs);
743       // TODO optimize by incrementally adding instead of reloading.
744     }
745 
746     // Make sure there are no holes now.
747     if (shouldFixHdfsHoles()) {
748       clearState(); // this also resets # fixes.
749       loadHdfsRegionDirs();
750       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
751       tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
752     }
753 
754     // Now we fix overlaps
755     if (shouldFixHdfsOverlaps()) {
756       // second pass we fix overlaps.
757       clearState(); // this also resets # fixes.
758       loadHdfsRegionDirs();
759       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
760       tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
761     }
762 
763     return errors.getErrorList().size();
764   }
765 
766   /**
767    * Scan all the store file names to find any lingering reference files,
768    * which refer to some none-exiting files. If "fix" option is enabled,
769    * any lingering reference file will be sidelined if found.
770    * <p>
771    * Lingering reference file prevents a region from opening. It has to
772    * be fixed before a cluster can start properly.
773    */
774   private void offlineReferenceFileRepair() throws IOException {
775     Configuration conf = getConf();
776     Path hbaseRoot = FSUtils.getRootDir(conf);
777     FileSystem fs = hbaseRoot.getFileSystem(conf);
778     Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot);
779     for (Path path: allFiles.values()) {
780       boolean isReference = false;
781       try {
782         isReference = StoreFileInfo.isReference(path);
783       } catch (Throwable t) {
784         // Ignore. Some files may not be store files at all.
785         // For example, files under .oldlogs folder in hbase:meta
786         // Warning message is already logged by
787         // StoreFile#isReference.
788       }
789       if (!isReference) continue;
790 
791       Path referredToFile = StoreFileInfo.getReferredToFile(path);
792       if (fs.exists(referredToFile)) continue;  // good, expected
793 
794       // Found a lingering reference file
795       errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
796         "Found lingering reference file " + path);
797       if (!shouldFixReferenceFiles()) continue;
798 
799       // Now, trying to fix it since requested
800       boolean success = false;
801       String pathStr = path.toString();
802 
803       // A reference file path should be like
804       // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
805       // Up 5 directories to get the root folder.
806       // So the file will be sidelined to a similar folder structure.
807       int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
808       for (int i = 0; index > 0 && i < 5; i++) {
809         index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
810       }
811       if (index > 0) {
812         Path rootDir = getSidelineDir();
813         Path dst = new Path(rootDir, pathStr.substring(index + 1));
814         fs.mkdirs(dst.getParent());
815         LOG.info("Trying to sildeline reference file "
816           + path + " to " + dst);
817         setShouldRerun();
818 
819         success = fs.rename(path, dst);
820       }
821       if (!success) {
822         LOG.error("Failed to sideline reference file " + path);
823       }
824     }
825   }
826 
827   /**
828    * TODO -- need to add tests for this.
829    */
830   private void reportEmptyMetaCells() {
831     errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
832       emptyRegionInfoQualifiers.size());
833     if (details) {
834       for (Result r: emptyRegionInfoQualifiers) {
835         errors.print("  " + r);
836       }
837     }
838   }
839 
840   /**
841    * TODO -- need to add tests for this.
842    */
843   private void reportTablesInFlux() {
844     AtomicInteger numSkipped = new AtomicInteger(0);
845     HTableDescriptor[] allTables = getTables(numSkipped);
846     errors.print("Number of Tables: " + allTables.length);
847     if (details) {
848       if (numSkipped.get() > 0) {
849         errors.detail("Number of Tables in flux: " + numSkipped.get());
850       }
851       for (HTableDescriptor td : allTables) {
852         errors.detail("  Table: " + td.getTableName() + "\t" +
853                            (td.isReadOnly() ? "ro" : "rw") + "\t" +
854                             (td.isMetaRegion() ? "META" : "    ") + "\t" +
855                            " families: " + td.getFamilies().size());
856       }
857     }
858   }
859 
860   public ErrorReporter getErrors() {
861     return errors;
862   }
863 
864   /**
865    * Read the .regioninfo file from the file system.  If there is no
866    * .regioninfo, add it to the orphan hdfs region list.
867    */
868   private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
869     Path regionDir = hbi.getHdfsRegionDir();
870     if (regionDir == null) {
871       LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
872       return;
873     }
874 
875     if (hbi.hdfsEntry.hri != null) {
876       // already loaded data
877       return;
878     }
879 
880     FileSystem fs = FileSystem.get(getConf());
881     HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
882     LOG.debug("HRegionInfo read: " + hri.toString());
883     hbi.hdfsEntry.hri = hri;
884   }
885 
886   /**
887    * Exception thrown when a integrity repair operation fails in an
888    * unresolvable way.
889    */
890   public static class RegionRepairException extends IOException {
891     private static final long serialVersionUID = 1L;
892     final IOException ioe;
893     public RegionRepairException(String s, IOException ioe) {
894       super(s);
895       this.ioe = ioe;
896     }
897   }
898 
899   /**
900    * Populate hbi's from regionInfos loaded from file system.
901    */
902   private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
903       throws IOException, InterruptedException {
904     tablesInfo.clear(); // regenerating the data
905     // generate region split structure
906     Collection<HbckInfo> hbckInfos = regionInfoMap.values();
907 
908     // Parallelized read of .regioninfo files.
909     List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
910     List<Future<Void>> hbiFutures;
911 
912     for (HbckInfo hbi : hbckInfos) {
913       WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
914       hbis.add(work);
915     }
916 
917     // Submit and wait for completion
918     hbiFutures = executor.invokeAll(hbis);
919 
920     for(int i=0; i<hbiFutures.size(); i++) {
921       WorkItemHdfsRegionInfo work = hbis.get(i);
922       Future<Void> f = hbiFutures.get(i);
923       try {
924         f.get();
925       } catch(ExecutionException e) {
926         LOG.warn("Failed to read .regioninfo file for region " +
927               work.hbi.getRegionNameAsString(), e.getCause());
928       }
929     }
930 
931     Path hbaseRoot = FSUtils.getRootDir(getConf());
932     FileSystem fs = hbaseRoot.getFileSystem(getConf());
933     // serialized table info gathering.
934     for (HbckInfo hbi: hbckInfos) {
935 
936       if (hbi.getHdfsHRI() == null) {
937         // was an orphan
938         continue;
939       }
940 
941 
942       // get table name from hdfs, populate various HBaseFsck tables.
943       TableName tableName = hbi.getTableName();
944       if (tableName == null) {
945         // There was an entry in hbase:meta not in the HDFS?
946         LOG.warn("tableName was null for: " + hbi);
947         continue;
948       }
949 
950       TableInfo modTInfo = tablesInfo.get(tableName);
951       if (modTInfo == null) {
952         // only executed once per table.
953         modTInfo = new TableInfo(tableName);
954         tablesInfo.put(tableName, modTInfo);
955         try {
956           TableDescriptor htd =
957               FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
958           modTInfo.htds.add(htd.getHTableDescriptor());
959         } catch (IOException ioe) {
960           if (!orphanTableDirs.containsKey(tableName)) {
961             LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
962             //should only report once for each table
963             errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
964                 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
965             Set<String> columns = new HashSet<String>();
966             orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
967           }
968         }
969       }
970       if (!hbi.isSkipChecks()) {
971         modTInfo.addRegionInfo(hbi);
972       }
973     }
974 
975     loadTableInfosForTablesWithNoRegion();
976 
977     return tablesInfo;
978   }
979 
980   /**
981    * To get the column family list according to the column family dirs
982    * @param columns
983    * @param hbi
984    * @return a set of column families
985    * @throws IOException
986    */
987   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
988     Path regionDir = hbi.getHdfsRegionDir();
989     FileSystem fs = regionDir.getFileSystem(getConf());
990     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
991     for (FileStatus subdir : subDirs) {
992       String columnfamily = subdir.getPath().getName();
993       columns.add(columnfamily);
994     }
995     return columns;
996   }
997 
998   /**
999    * To fabricate a .tableinfo file with following contents<br>
1000    * 1. the correct tablename <br>
1001    * 2. the correct colfamily list<br>
1002    * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1003    * @throws IOException
1004    */
1005   private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1006       Set<String> columns) throws IOException {
1007     if (columns ==null || columns.isEmpty()) return false;
1008     HTableDescriptor htd = new HTableDescriptor(tableName);
1009     for (String columnfamimly : columns) {
1010       htd.addFamily(new HColumnDescriptor(columnfamimly));
1011     }
1012     fstd.createTableDescriptor(new TableDescriptor(htd, TableState.State.ENABLED), true);
1013     return true;
1014   }
1015 
1016   /**
1017    * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1018    * @throws IOException
1019    */
1020   public void fixEmptyMetaCells() throws IOException {
1021     if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1022       LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1023       for (Result region : emptyRegionInfoQualifiers) {
1024         deleteMetaRegion(region.getRow());
1025         errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1026       }
1027       emptyRegionInfoQualifiers.clear();
1028     }
1029   }
1030 
1031   /**
1032    * To fix orphan table by creating a .tableinfo file under tableDir <br>
1033    * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1034    * 2. else create a default .tableinfo file with following items<br>
1035    * &nbsp;2.1 the correct tablename <br>
1036    * &nbsp;2.2 the correct colfamily list<br>
1037    * &nbsp;2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1038    * @throws IOException
1039    */
1040   public void fixOrphanTables() throws IOException {
1041     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1042 
1043       List<TableName> tmpList = new ArrayList<TableName>();
1044       tmpList.addAll(orphanTableDirs.keySet());
1045       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1046       Iterator<Entry<TableName, Set<String>>> iter =
1047           orphanTableDirs.entrySet().iterator();
1048       int j = 0;
1049       int numFailedCase = 0;
1050       FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1051       while (iter.hasNext()) {
1052         Entry<TableName, Set<String>> entry =
1053             (Entry<TableName, Set<String>>) iter.next();
1054         TableName tableName = entry.getKey();
1055         LOG.info("Trying to fix orphan table error: " + tableName);
1056         if (j < htds.length) {
1057           if (tableName.equals(htds[j].getTableName())) {
1058             HTableDescriptor htd = htds[j];
1059             LOG.info("fixing orphan table: " + tableName + " from cache");
1060             fstd.createTableDescriptor(new TableDescriptor(htd, TableState.State.ENABLED), true);
1061             j++;
1062             iter.remove();
1063           }
1064         } else {
1065           if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1066             LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1067             LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1068             iter.remove();
1069           } else {
1070             LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1071             numFailedCase++;
1072           }
1073         }
1074         fixes++;
1075       }
1076 
1077       if (orphanTableDirs.isEmpty()) {
1078         // all orphanTableDirs are luckily recovered
1079         // re-run doFsck after recovering the .tableinfo file
1080         setShouldRerun();
1081         LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1082       } else if (numFailedCase > 0) {
1083         LOG.error("Failed to fix " + numFailedCase
1084             + " OrphanTables with default .tableinfo files");
1085       }
1086 
1087     }
1088     //cleanup the list
1089     orphanTableDirs.clear();
1090 
1091   }
1092 
1093   /**
1094    * This borrows code from MasterFileSystem.bootstrap()
1095    *
1096    * @return an open hbase:meta HRegion
1097    */
1098   private HRegion createNewMeta() throws IOException {
1099       Path rootdir = FSUtils.getRootDir(getConf());
1100     Configuration c = getConf();
1101     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1102     MasterFileSystem.setInfoFamilyCachingForMeta(false);
1103     HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c,
1104         HTableDescriptor.META_TABLEDESC);
1105     MasterFileSystem.setInfoFamilyCachingForMeta(true);
1106     return meta;
1107   }
1108 
1109   /**
1110    * Generate set of puts to add to new meta.  This expects the tables to be
1111    * clean with no overlaps or holes.  If there are any problems it returns null.
1112    *
1113    * @return An array list of puts to do in bulk, null if tables have problems
1114    */
1115   private ArrayList<Put> generatePuts(
1116       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1117     ArrayList<Put> puts = new ArrayList<Put>();
1118     boolean hasProblems = false;
1119     for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1120       TableName name = e.getKey();
1121 
1122       // skip "hbase:meta"
1123       if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1124         continue;
1125       }
1126 
1127       TableInfo ti = e.getValue();
1128       for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1129           .entrySet()) {
1130         Collection<HbckInfo> his = spl.getValue();
1131         int sz = his.size();
1132         if (sz != 1) {
1133           // problem
1134           LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1135               + " had " +  sz + " regions instead of exactly 1." );
1136           hasProblems = true;
1137           continue;
1138         }
1139 
1140         // add the row directly to meta.
1141         HbckInfo hi = his.iterator().next();
1142         HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
1143         Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1144         puts.add(p);
1145       }
1146     }
1147     return hasProblems ? null : puts;
1148   }
1149 
1150   /**
1151    * Suggest fixes for each table
1152    */
1153   private void suggestFixes(
1154       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1155     for (TableInfo tInfo : tablesInfo.values()) {
1156       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1157       tInfo.checkRegionChain(handler);
1158     }
1159   }
1160 
1161   /**
1162    * Rebuilds meta from information in hdfs/fs.  Depends on configuration
1163    * settings passed into hbck constructor to point to a particular fs/dir.
1164    *
1165    * @param fix flag that determines if method should attempt to fix holes
1166    * @return true if successful, false if attempt failed.
1167    */
1168   public boolean rebuildMeta(boolean fix) throws IOException,
1169       InterruptedException {
1170 
1171     // TODO check to make sure hbase is offline. (or at least the table
1172     // currently being worked on is off line)
1173 
1174     // Determine what's on HDFS
1175     LOG.info("Loading HBase regioninfo from HDFS...");
1176     loadHdfsRegionDirs(); // populating regioninfo table.
1177 
1178     int errs = errors.getErrorList().size();
1179     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1180     checkHdfsIntegrity(false, false);
1181 
1182     // make sure ok.
1183     if (errors.getErrorList().size() != errs) {
1184       // While in error state, iterate until no more fixes possible
1185       while(true) {
1186         fixes = 0;
1187         suggestFixes(tablesInfo);
1188         errors.clear();
1189         loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1190         checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1191 
1192         int errCount = errors.getErrorList().size();
1193 
1194         if (fixes == 0) {
1195           if (errCount > 0) {
1196             return false; // failed to fix problems.
1197           } else {
1198             break; // no fixes and no problems? drop out and fix stuff!
1199           }
1200         }
1201       }
1202     }
1203 
1204     // we can rebuild, move old meta out of the way and start
1205     LOG.info("HDFS regioninfo's seems good.  Sidelining old hbase:meta");
1206     Path backupDir = sidelineOldMeta();
1207 
1208     LOG.info("Creating new hbase:meta");
1209     HRegion meta = createNewMeta();
1210 
1211     // populate meta
1212     List<Put> puts = generatePuts(tablesInfo);
1213     if (puts == null) {
1214       LOG.fatal("Problem encountered when creating new hbase:meta entries.  " +
1215         "You may need to restore the previously sidelined hbase:meta");
1216       return false;
1217     }
1218     meta.batchMutate(puts.toArray(new Put[0]));
1219     HRegion.closeHRegion(meta);
1220     LOG.info("Success! hbase:meta table rebuilt.");
1221     LOG.info("Old hbase:meta is moved into " + backupDir);
1222     return true;
1223   }
1224 
1225   private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1226       boolean fixOverlaps) throws IOException {
1227     LOG.info("Checking HBase region split map from HDFS data...");
1228     for (TableInfo tInfo : tablesInfo.values()) {
1229       TableIntegrityErrorHandler handler;
1230       if (fixHoles || fixOverlaps) {
1231         handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1232           fixHoles, fixOverlaps);
1233       } else {
1234         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1235       }
1236       if (!tInfo.checkRegionChain(handler)) {
1237         // should dump info as well.
1238         errors.report("Found inconsistency in table " + tInfo.getName());
1239       }
1240     }
1241     return tablesInfo;
1242   }
1243 
1244   private Path getSidelineDir() throws IOException {
1245     if (sidelineDir == null) {
1246       Path hbaseDir = FSUtils.getRootDir(getConf());
1247       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1248       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1249           + startMillis);
1250     }
1251     return sidelineDir;
1252   }
1253 
1254   /**
1255    * Sideline a region dir (instead of deleting it)
1256    */
1257   Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1258     return sidelineRegionDir(fs, null, hi);
1259   }
1260 
1261   /**
1262    * Sideline a region dir (instead of deleting it)
1263    *
1264    * @param parentDir if specified, the region will be sidelined to
1265    * folder like .../parentDir/<table name>/<region name>. The purpose
1266    * is to group together similar regions sidelined, for example, those
1267    * regions should be bulk loaded back later on. If null, it is ignored.
1268    */
1269   Path sidelineRegionDir(FileSystem fs,
1270       String parentDir, HbckInfo hi) throws IOException {
1271     TableName tableName = hi.getTableName();
1272     Path regionDir = hi.getHdfsRegionDir();
1273 
1274     if (!fs.exists(regionDir)) {
1275       LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1276       return null;
1277     }
1278 
1279     Path rootDir = getSidelineDir();
1280     if (parentDir != null) {
1281       rootDir = new Path(rootDir, parentDir);
1282     }
1283     Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1284     Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1285     fs.mkdirs(sidelineRegionDir);
1286     boolean success = false;
1287     FileStatus[] cfs =  fs.listStatus(regionDir);
1288     if (cfs == null) {
1289       LOG.info("Region dir is empty: " + regionDir);
1290     } else {
1291       for (FileStatus cf : cfs) {
1292         Path src = cf.getPath();
1293         Path dst =  new Path(sidelineRegionDir, src.getName());
1294         if (fs.isFile(src)) {
1295           // simple file
1296           success = fs.rename(src, dst);
1297           if (!success) {
1298             String msg = "Unable to rename file " + src +  " to " + dst;
1299             LOG.error(msg);
1300             throw new IOException(msg);
1301           }
1302           continue;
1303         }
1304 
1305         // is a directory.
1306         fs.mkdirs(dst);
1307 
1308         LOG.info("Sidelining files from " + src + " into containing region " + dst);
1309         // FileSystem.rename is inconsistent with directories -- if the
1310         // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1311         // it moves the src into the dst dir resulting in (foo/a/b).  If
1312         // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1313         FileStatus[] hfiles = fs.listStatus(src);
1314         if (hfiles != null && hfiles.length > 0) {
1315           for (FileStatus hfile : hfiles) {
1316             success = fs.rename(hfile.getPath(), dst);
1317             if (!success) {
1318               String msg = "Unable to rename file " + src +  " to " + dst;
1319               LOG.error(msg);
1320               throw new IOException(msg);
1321             }
1322           }
1323         }
1324         LOG.debug("Sideline directory contents:");
1325         debugLsr(sidelineRegionDir);
1326       }
1327     }
1328 
1329     LOG.info("Removing old region dir: " + regionDir);
1330     success = fs.delete(regionDir, true);
1331     if (!success) {
1332       String msg = "Unable to delete dir " + regionDir;
1333       LOG.error(msg);
1334       throw new IOException(msg);
1335     }
1336     return sidelineRegionDir;
1337   }
1338 
1339   /**
1340    * Side line an entire table.
1341    */
1342   void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1343       Path backupHbaseDir) throws IOException {
1344     Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1345     if (fs.exists(tableDir)) {
1346       Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1347       fs.mkdirs(backupTableDir.getParent());
1348       boolean success = fs.rename(tableDir, backupTableDir);
1349       if (!success) {
1350         throw new IOException("Failed to move  " + tableName + " from "
1351             +  tableDir + " to " + backupTableDir);
1352       }
1353     } else {
1354       LOG.info("No previous " + tableName +  " exists.  Continuing.");
1355     }
1356   }
1357 
1358   /**
1359    * @return Path to backup of original directory
1360    */
1361   Path sidelineOldMeta() throws IOException {
1362     // put current hbase:meta aside.
1363     Path hbaseDir = FSUtils.getRootDir(getConf());
1364     FileSystem fs = hbaseDir.getFileSystem(getConf());
1365     Path backupDir = getSidelineDir();
1366     fs.mkdirs(backupDir);
1367 
1368     try {
1369       sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1370     } catch (IOException e) {
1371         LOG.fatal("... failed to sideline meta. Currently in inconsistent state.  To restore "
1372             + "try to rename hbase:meta in " + backupDir.getName() + " to "
1373             + hbaseDir.getName() + ".", e);
1374       throw e; // throw original exception
1375     }
1376     return backupDir;
1377   }
1378 
1379   /**
1380    * Load the list of disabled tables in ZK into local set.
1381    * @throws ZooKeeperConnectionException
1382    * @throws IOException
1383    */
1384   private void loadDisabledTables()
1385   throws IOException {
1386     HConnectionManager.execute(new HConnectable<Void>(getConf()) {
1387       @Override
1388       public Void connect(HConnection connection) throws IOException {
1389         TableName[] tables = connection.listTableNames();
1390         for (TableName table : tables) {
1391           if (connection.getTableState(table)
1392               .inStates(TableState.State.DISABLED, TableState.State.DISABLING)) {
1393             disabledTables.add(table);
1394           }
1395         }
1396         return null;
1397       }
1398     });
1399   }
1400 
1401   /**
1402    * Check if the specified region's table is disabled.
1403    */
1404   private boolean isTableDisabled(HRegionInfo regionInfo) {
1405     return disabledTables.contains(regionInfo.getTable());
1406   }
1407 
1408   /**
1409    * Scan HDFS for all regions, recording their information into
1410    * regionInfoMap
1411    */
1412   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1413     Path rootDir = FSUtils.getRootDir(getConf());
1414     FileSystem fs = rootDir.getFileSystem(getConf());
1415 
1416     // list all tables from HDFS
1417     List<FileStatus> tableDirs = Lists.newArrayList();
1418 
1419     boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1420 
1421     List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1422     for (Path path : paths) {
1423       TableName tableName = FSUtils.getTableName(path);
1424        if ((!checkMetaOnly &&
1425            isTableIncluded(tableName)) ||
1426            tableName.equals(TableName.META_TABLE_NAME)) {
1427          tableDirs.add(fs.getFileStatus(path));
1428        }
1429     }
1430 
1431     // verify that version file exists
1432     if (!foundVersionFile) {
1433       errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1434           "Version file does not exist in root dir " + rootDir);
1435       if (shouldFixVersionFile()) {
1436         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1437             + " file.");
1438         setShouldRerun();
1439         FSUtils.setVersion(fs, rootDir, getConf().getInt(
1440             HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1441             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1442             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1443       }
1444     }
1445 
1446     // level 1:  <HBASE_DIR>/*
1447     List<WorkItemHdfsDir> dirs = new ArrayList<WorkItemHdfsDir>(tableDirs.size());
1448     List<Future<Void>> dirsFutures;
1449 
1450     for (FileStatus tableDir : tableDirs) {
1451       LOG.debug("Loading region dirs from " +tableDir.getPath());
1452       dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir));
1453     }
1454 
1455     // Invoke and wait for Callables to complete
1456     dirsFutures = executor.invokeAll(dirs);
1457 
1458     for(Future<Void> f: dirsFutures) {
1459       try {
1460         f.get();
1461       } catch(ExecutionException e) {
1462         LOG.warn("Could not load region dir " , e.getCause());
1463       }
1464     }
1465   }
1466 
1467   /**
1468    * Record the location of the hbase:meta region as found in ZooKeeper.
1469    */
1470   private boolean recordMetaRegion() throws IOException {
1471     HRegionLocation metaLocation = connection.locateRegion(
1472       TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW);
1473 
1474     // Check if Meta region is valid and existing
1475     if (metaLocation == null || metaLocation.getRegionInfo() == null ||
1476         metaLocation.getHostname() == null) {
1477       errors.reportError(ERROR_CODE.NULL_META_REGION,
1478         "META region or some of its attributes are null.");
1479       return false;
1480     }
1481     ServerName sn;
1482     try {
1483       sn = getMetaRegionServerName();
1484     } catch (KeeperException e) {
1485       throw new IOException(e);
1486     }
1487     MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, System.currentTimeMillis());
1488     HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1489     if (hbckInfo == null) {
1490       regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1491     } else {
1492       hbckInfo.metaEntry = m;
1493     }
1494     return true;
1495   }
1496 
1497   private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1498     return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1499       @Override
1500       public void abort(String why, Throwable e) {
1501         LOG.error(why, e);
1502         System.exit(1);
1503       }
1504 
1505       @Override
1506       public boolean isAborted() {
1507         return false;
1508       }
1509 
1510     });
1511   }
1512 
1513   private ServerName getMetaRegionServerName()
1514   throws IOException, KeeperException {
1515     ZooKeeperWatcher zkw = createZooKeeperWatcher();
1516     ServerName sn = null;
1517     try {
1518       sn = new MetaTableLocator().getMetaRegionLocation(zkw);
1519     } finally {
1520       zkw.close();
1521     }
1522     return sn;
1523   }
1524 
1525   /**
1526    * Contacts each regionserver and fetches metadata about regions.
1527    * @param regionServerList - the list of region servers to connect to
1528    * @throws IOException if a remote or network exception occurs
1529    */
1530   void processRegionServers(Collection<ServerName> regionServerList)
1531     throws IOException, InterruptedException {
1532 
1533     List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1534     List<Future<Void>> workFutures;
1535 
1536     // loop to contact each region server in parallel
1537     for (ServerName rsinfo: regionServerList) {
1538       workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1539     }
1540 
1541     workFutures = executor.invokeAll(workItems);
1542 
1543     for(int i=0; i<workFutures.size(); i++) {
1544       WorkItemRegion item = workItems.get(i);
1545       Future<Void> f = workFutures.get(i);
1546       try {
1547         f.get();
1548       } catch(ExecutionException e) {
1549         LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1550             e.getCause());
1551       }
1552     }
1553   }
1554 
1555   /**
1556    * Check consistency of all regions that have been found in previous phases.
1557    */
1558   private void checkAndFixConsistency()
1559   throws IOException, KeeperException, InterruptedException {
1560 	  // Divide the checks in two phases. One for default/primary replicas and another
1561 	  // for the non-primary ones. Keeps code cleaner this way.
1562     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1563       if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1564         checkRegionConsistency(e.getKey(), e.getValue());
1565       }
1566     }
1567     boolean prevHdfsCheck = shouldCheckHdfs();
1568     setCheckHdfs(false); //replicas don't have any hdfs data
1569     // Run a pass over the replicas and fix any assignment issues that exist on the currently
1570     // deployed/undeployed replicas.
1571     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1572       if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1573         checkRegionConsistency(e.getKey(), e.getValue());
1574       }
1575     }
1576     setCheckHdfs(prevHdfsCheck);
1577   }
1578 
1579   private void preCheckPermission() throws IOException, AccessDeniedException {
1580     if (shouldIgnorePreCheckPermission()) {
1581       return;
1582     }
1583 
1584     Path hbaseDir = FSUtils.getRootDir(getConf());
1585     FileSystem fs = hbaseDir.getFileSystem(getConf());
1586     UserProvider userProvider = UserProvider.instantiate(getConf());
1587     UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1588     FileStatus[] files = fs.listStatus(hbaseDir);
1589     for (FileStatus file : files) {
1590       try {
1591         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1592       } catch (AccessDeniedException ace) {
1593         LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1594         errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1595           + " does not have write perms to " + file.getPath()
1596           + ". Please rerun hbck as hdfs user " + file.getOwner());
1597         throw ace;
1598       }
1599     }
1600   }
1601 
1602   /**
1603    * Deletes region from meta table
1604    */
1605   private void deleteMetaRegion(HbckInfo hi) throws IOException {
1606     deleteMetaRegion(hi.metaEntry.getRegionName());
1607   }
1608 
1609   /**
1610    * Deletes region from meta table
1611    */
1612   private void deleteMetaRegion(byte[] metaKey) throws IOException {
1613     Delete d = new Delete(metaKey);
1614     meta.delete(d);
1615     meta.flushCommits();
1616     LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1617   }
1618 
1619   /**
1620    * Reset the split parent region info in meta table
1621    */
1622   private void resetSplitParent(HbckInfo hi) throws IOException {
1623     RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
1624     Delete d = new Delete(hi.metaEntry.getRegionName());
1625     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1626     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1627     mutations.add(d);
1628 
1629     HRegionInfo hri = new HRegionInfo(hi.metaEntry);
1630     hri.setOffline(false);
1631     hri.setSplit(false);
1632     Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1633     mutations.add(p);
1634 
1635     meta.mutateRow(mutations);
1636     meta.flushCommits();
1637     LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
1638   }
1639 
1640   /**
1641    * This backwards-compatibility wrapper for permanently offlining a region
1642    * that should not be alive.  If the region server does not support the
1643    * "offline" method, it will use the closest unassign method instead.  This
1644    * will basically work until one attempts to disable or delete the affected
1645    * table.  The problem has to do with in-memory only master state, so
1646    * restarting the HMaster or failing over to another should fix this.
1647    */
1648   private void offline(byte[] regionName) throws IOException {
1649     String regionString = Bytes.toStringBinary(regionName);
1650     if (!rsSupportsOffline) {
1651       LOG.warn("Using unassign region " + regionString
1652           + " instead of using offline method, you should"
1653           + " restart HMaster after these repairs");
1654       admin.unassign(regionName, true);
1655       return;
1656     }
1657 
1658     // first time we assume the rs's supports #offline.
1659     try {
1660       LOG.info("Offlining region " + regionString);
1661       admin.offline(regionName);
1662     } catch (IOException ioe) {
1663       String notFoundMsg = "java.lang.NoSuchMethodException: " +
1664         "org.apache.hadoop.hbase.master.HMaster.offline([B)";
1665       if (ioe.getMessage().contains(notFoundMsg)) {
1666         LOG.warn("Using unassign region " + regionString
1667             + " instead of using offline method, you should"
1668             + " restart HMaster after these repairs");
1669         rsSupportsOffline = false; // in the future just use unassign
1670         admin.unassign(regionName, true);
1671         return;
1672       }
1673       throw ioe;
1674     }
1675   }
1676 
1677   private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
1678     undeployRegionsForHbi(hi);
1679     // undeploy replicas of the region (but only if the method is invoked for the primary)
1680     if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1681       return;
1682     }
1683     int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
1684     for (int i = 1; i < numReplicas; i++) {
1685       if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
1686       HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
1687           hi.getPrimaryHRIForDeployedReplica(), i);
1688       HbckInfo h = regionInfoMap.get(hri.getEncodedName());
1689       if (h != null) {
1690         undeployRegionsForHbi(h);
1691         //set skip checks; we undeployed it, and we don't want to evaluate this anymore
1692         //in consistency checks
1693         h.setSkipChecks(true);
1694       }
1695     }
1696   }
1697 
1698   private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
1699     for (OnlineEntry rse : hi.deployedEntries) {
1700       LOG.debug("Undeploy region "  + rse.hri + " from " + rse.hsa);
1701       try {
1702         HBaseFsckRepair.closeRegionSilentlyAndWait(admin, rse.hsa, rse.hri);
1703         offline(rse.hri.getRegionName());
1704       } catch (IOException ioe) {
1705         LOG.warn("Got exception when attempting to offline region "
1706             + Bytes.toString(rse.hri.getRegionName()), ioe);
1707       }
1708     }
1709   }
1710 
1711   /**
1712    * Attempts to undeploy a region from a region server based in information in
1713    * META.  Any operations that modify the file system should make sure that
1714    * its corresponding region is not deployed to prevent data races.
1715    *
1716    * A separate call is required to update the master in-memory region state
1717    * kept in the AssignementManager.  Because disable uses this state instead of
1718    * that found in META, we can't seem to cleanly disable/delete tables that
1719    * have been hbck fixed.  When used on a version of HBase that does not have
1720    * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master
1721    * restart or failover may be required.
1722    */
1723   private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
1724     if (hi.metaEntry == null && hi.hdfsEntry == null) {
1725       undeployRegions(hi);
1726       return;
1727     }
1728 
1729     // get assignment info and hregioninfo from meta.
1730     Get get = new Get(hi.getRegionName());
1731     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1732     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
1733     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
1734     // also get the locations of the replicas to close if the primary region is being closed
1735     if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1736       int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
1737       for (int i = 0; i < numReplicas; i++) {
1738         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
1739         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
1740       }
1741     }
1742     Result r = meta.get(get);
1743     RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
1744     if (rl == null) {
1745       LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
1746           " since meta does not have handle to reach it");
1747       return;
1748     }
1749     for (HRegionLocation h : rl.getRegionLocations()) {
1750       ServerName serverName = h.getServerName();
1751       if (serverName == null) {
1752         errors.reportError("Unable to close region "
1753             + hi.getRegionNameAsString() +  " because meta does not "
1754             + "have handle to reach it.");
1755         continue;
1756       }
1757       HRegionInfo hri = h.getRegionInfo();
1758       if (hri == null) {
1759         LOG.warn("Unable to close region " + hi.getRegionNameAsString()
1760             + " because hbase:meta had invalid or missing "
1761             + HConstants.CATALOG_FAMILY_STR + ":"
1762             + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
1763             + " qualifier value.");
1764         continue;
1765       }
1766       // close the region -- close files and remove assignment
1767       HBaseFsckRepair.closeRegionSilentlyAndWait(admin, serverName, hri);
1768     }
1769   }
1770 
1771   private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
1772     KeeperException, InterruptedException {
1773     // If we are trying to fix the errors
1774     if (shouldFixAssignments()) {
1775       errors.print(msg);
1776       undeployRegions(hbi);
1777       setShouldRerun();
1778       HRegionInfo hri = hbi.getHdfsHRI();
1779       if (hri == null) {
1780         hri = hbi.metaEntry;
1781       }
1782       HBaseFsckRepair.fixUnassigned(admin, hri);
1783       HBaseFsckRepair.waitUntilAssigned(admin, hri);
1784 
1785       // also assign replicas if needed (do it only when this call operates on a primary replica)
1786       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
1787       int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
1788       for (int i = 1; i < replicationCount; i++) {
1789         hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
1790         HbckInfo h = regionInfoMap.get(hri.getEncodedName());
1791         if (h != null) {
1792           undeployRegions(h);
1793           //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore
1794           //in consistency checks
1795           h.setSkipChecks(true);
1796         }
1797         HBaseFsckRepair.fixUnassigned(admin, hri);
1798         HBaseFsckRepair.waitUntilAssigned(admin, hri);
1799       }
1800 
1801     }
1802   }
1803 
1804   /**
1805    * Check a single region for consistency and correct deployment.
1806    */
1807   private void checkRegionConsistency(final String key, final HbckInfo hbi)
1808   throws IOException, KeeperException, InterruptedException {
1809 
1810 	if (hbi.isSkipChecks()) return;
1811 	String descriptiveName = hbi.toString();
1812     boolean inMeta = hbi.metaEntry != null;
1813     // In case not checking HDFS, assume the region is on HDFS
1814     boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
1815     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
1816     boolean isDeployed = !hbi.deployedOn.isEmpty();
1817     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
1818     boolean deploymentMatchesMeta =
1819       hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
1820       hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
1821     boolean splitParent =
1822       (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
1823     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
1824     boolean recentlyModified = inHdfs &&
1825       hbi.getModTime() + timelag > System.currentTimeMillis();
1826 
1827     // ========== First the healthy cases =============
1828     if (hbi.containsOnlyHdfsEdits()) {
1829       return;
1830     }
1831     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
1832       return;
1833     } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
1834       LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
1835         "tabled that is not deployed");
1836       return;
1837     } else if (recentlyModified) {
1838       LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
1839       return;
1840     }
1841     // ========== Cases where the region is not in hbase:meta =============
1842     else if (!inMeta && !inHdfs && !isDeployed) {
1843       // We shouldn't have record of this region at all then!
1844       assert false : "Entry for region with no data";
1845     } else if (!inMeta && !inHdfs && isDeployed) {
1846       errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
1847           + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
1848           "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1849       if (shouldFixAssignments()) {
1850         undeployRegions(hbi);
1851       }
1852 
1853     } else if (!inMeta && inHdfs && !isDeployed) {
1854       if (hbi.isMerged()) {
1855         // This region has already been merged, the remaining hdfs file will be
1856         // cleaned by CatalogJanitor later
1857         hbi.setSkipChecks(true);
1858         LOG.info("Region " + descriptiveName
1859             + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
1860         return;
1861       }
1862       errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
1863           + descriptiveName + " on HDFS, but not listed in hbase:meta " +
1864           "or deployed on any region server");
1865       // restore region consistency of an adopted orphan
1866       if (shouldFixMeta()) {
1867         if (!hbi.isHdfsRegioninfoPresent()) {
1868           LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
1869               +  " in table integrity repair phase if -fixHdfsOrphans was" +
1870               " used.");
1871           return;
1872         }
1873 
1874         LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
1875         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
1876         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
1877             admin.getClusterStatus().getServers(), numReplicas);
1878 
1879         tryAssignmentRepair(hbi, "Trying to reassign region...");
1880       }
1881 
1882     } else if (!inMeta && inHdfs && isDeployed) {
1883       errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
1884           + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1885       debugLsr(hbi.getHdfsRegionDir());
1886       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1887         // for replicas, this means that we should undeploy the region (we would have
1888         // gone over the primaries and fixed meta holes in first phase under
1889         // checkAndFixConsistency; we shouldn't get the condition !inMeta at
1890         // this stage unless unwanted replica)
1891         if (shouldFixAssignments()) {
1892           undeployRegionsForHbi(hbi);
1893         }
1894       }
1895       if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1896         if (!hbi.isHdfsRegioninfoPresent()) {
1897           LOG.error("This should have been repaired in table integrity repair phase");
1898           return;
1899         }
1900 
1901         LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
1902         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
1903         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
1904             admin.getClusterStatus().getServers(), numReplicas);
1905         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
1906       }
1907 
1908     // ========== Cases where the region is in hbase:meta =============
1909     } else if (inMeta && inHdfs && !isDeployed && splitParent) {
1910       // check whether this is an actual error, or just transient state where parent
1911       // is not cleaned
1912       if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
1913         // check that split daughters are there
1914         HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
1915         HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
1916         if (infoA != null && infoB != null) {
1917           // we already processed or will process daughters. Move on, nothing to see here.
1918           hbi.setSkipChecks(true);
1919           return;
1920         }
1921       }
1922       errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
1923           + descriptiveName + " is a split parent in META, in HDFS, "
1924           + "and not deployed on any region server. This could be transient.");
1925       if (shouldFixSplitParents()) {
1926         setShouldRerun();
1927         resetSplitParent(hbi);
1928       }
1929     } else if (inMeta && !inHdfs && !isDeployed) {
1930       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
1931           + descriptiveName + " found in META, but not in HDFS "
1932           + "or deployed on any region server.");
1933       if (shouldFixMeta()) {
1934         deleteMetaRegion(hbi);
1935       }
1936     } else if (inMeta && !inHdfs && isDeployed) {
1937       errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
1938           + " found in META, but not in HDFS, " +
1939           "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1940       // We treat HDFS as ground truth.  Any information in meta is transient
1941       // and equivalent data can be regenerated.  So, lets unassign and remove
1942       // these problems from META.
1943       if (shouldFixAssignments()) {
1944         errors.print("Trying to fix unassigned region...");
1945         undeployRegions(hbi);
1946       }
1947       if (shouldFixMeta()) {
1948         // wait for it to complete
1949         deleteMetaRegion(hbi);
1950       }
1951     } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
1952       errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
1953           + " not deployed on any region server.");
1954       tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
1955     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
1956       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
1957           "Region " + descriptiveName + " should not be deployed according " +
1958           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1959       if (shouldFixAssignments()) {
1960         errors.print("Trying to close the region " + descriptiveName);
1961         setShouldRerun();
1962         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
1963       }
1964     } else if (inMeta && inHdfs && isMultiplyDeployed) {
1965       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
1966           + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
1967           + " but is multiply assigned to region servers " +
1968           Joiner.on(", ").join(hbi.deployedOn));
1969       // If we are trying to fix the errors
1970       if (shouldFixAssignments()) {
1971         errors.print("Trying to fix assignment error...");
1972         setShouldRerun();
1973         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
1974       }
1975     } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
1976       errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
1977           + descriptiveName + " listed in hbase:meta on region server " +
1978           hbi.metaEntry.regionServer + " but found on region server " +
1979           hbi.deployedOn.get(0));
1980       // If we are trying to fix the errors
1981       if (shouldFixAssignments()) {
1982         errors.print("Trying to fix assignment error...");
1983         setShouldRerun();
1984         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
1985         HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
1986       }
1987     } else {
1988       errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
1989           " is in an unforeseen state:" +
1990           " inMeta=" + inMeta +
1991           " inHdfs=" + inHdfs +
1992           " isDeployed=" + isDeployed +
1993           " isMultiplyDeployed=" + isMultiplyDeployed +
1994           " deploymentMatchesMeta=" + deploymentMatchesMeta +
1995           " shouldBeDeployed=" + shouldBeDeployed);
1996     }
1997   }
1998 
1999   /**
2000    * Checks tables integrity. Goes over all regions and scans the tables.
2001    * Collects all the pieces for each table and checks if there are missing,
2002    * repeated or overlapping ones.
2003    * @throws IOException
2004    */
2005   SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2006     tablesInfo = new TreeMap<TableName,TableInfo> ();
2007     LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2008     for (HbckInfo hbi : regionInfoMap.values()) {
2009       // Check only valid, working regions
2010       if (hbi.metaEntry == null) {
2011         // this assumes that consistency check has run loadMetaEntry
2012         Path p = hbi.getHdfsRegionDir();
2013         if (p == null) {
2014           errors.report("No regioninfo in Meta or HDFS. " + hbi);
2015         }
2016 
2017         // TODO test.
2018         continue;
2019       }
2020       if (hbi.metaEntry.regionServer == null) {
2021         errors.detail("Skipping region because no region server: " + hbi);
2022         continue;
2023       }
2024       if (hbi.metaEntry.isOffline()) {
2025         errors.detail("Skipping region because it is offline: " + hbi);
2026         continue;
2027       }
2028       if (hbi.containsOnlyHdfsEdits()) {
2029         errors.detail("Skipping region because it only contains edits" + hbi);
2030         continue;
2031       }
2032 
2033       // Missing regionDir or over-deployment is checked elsewhere. Include
2034       // these cases in modTInfo, so we can evaluate those regions as part of
2035       // the region chain in META
2036       //if (hbi.foundRegionDir == null) continue;
2037       //if (hbi.deployedOn.size() != 1) continue;
2038       if (hbi.deployedOn.size() == 0) continue;
2039 
2040       // We should be safe here
2041       TableName tableName = hbi.metaEntry.getTable();
2042       TableInfo modTInfo = tablesInfo.get(tableName);
2043       if (modTInfo == null) {
2044         modTInfo = new TableInfo(tableName);
2045       }
2046       for (ServerName server : hbi.deployedOn) {
2047         modTInfo.addServer(server);
2048       }
2049 
2050       if (!hbi.isSkipChecks()) {
2051         modTInfo.addRegionInfo(hbi);
2052       }
2053 
2054       tablesInfo.put(tableName, modTInfo);
2055     }
2056 
2057     loadTableInfosForTablesWithNoRegion();
2058 
2059     for (TableInfo tInfo : tablesInfo.values()) {
2060       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2061       if (!tInfo.checkRegionChain(handler)) {
2062         errors.report("Found inconsistency in table " + tInfo.getName());
2063       }
2064     }
2065     return tablesInfo;
2066   }
2067 
2068   /** Loads table info's for tables that may not have been included, since there are no
2069    * regions reported for the table, but table dir is there in hdfs
2070    */
2071   private void loadTableInfosForTablesWithNoRegion() throws IOException {
2072     Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2073     for (HTableDescriptor htd : allTables.values()) {
2074       if (checkMetaOnly && !htd.isMetaTable()) {
2075         continue;
2076       }
2077 
2078       TableName tableName = htd.getTableName();
2079       if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2080         TableInfo tableInfo = new TableInfo(tableName);
2081         tableInfo.htds.add(htd);
2082         tablesInfo.put(htd.getTableName(), tableInfo);
2083       }
2084     }
2085   }
2086 
2087   /**
2088    * Merge hdfs data by moving from contained HbckInfo into targetRegionDir.
2089    * @return number of file move fixes done to merge regions.
2090    */
2091   public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2092     int fileMoves = 0;
2093     String thread = Thread.currentThread().getName();
2094     LOG.debug("[" + thread + "] Contained region dir after close and pause");
2095     debugLsr(contained.getHdfsRegionDir());
2096 
2097     // rename the contained into the container.
2098     FileSystem fs = targetRegionDir.getFileSystem(getConf());
2099     FileStatus[] dirs = null;
2100     try {
2101       dirs = fs.listStatus(contained.getHdfsRegionDir());
2102     } catch (FileNotFoundException fnfe) {
2103       // region we are attempting to merge in is not present!  Since this is a merge, there is
2104       // no harm skipping this region if it does not exist.
2105       if (!fs.exists(contained.getHdfsRegionDir())) {
2106         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2107             + " is missing. Assuming already sidelined or moved.");
2108       } else {
2109         sidelineRegionDir(fs, contained);
2110       }
2111       return fileMoves;
2112     }
2113 
2114     if (dirs == null) {
2115       if (!fs.exists(contained.getHdfsRegionDir())) {
2116         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2117             + " already sidelined.");
2118       } else {
2119         sidelineRegionDir(fs, contained);
2120       }
2121       return fileMoves;
2122     }
2123 
2124     for (FileStatus cf : dirs) {
2125       Path src = cf.getPath();
2126       Path dst =  new Path(targetRegionDir, src.getName());
2127 
2128       if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2129         // do not copy the old .regioninfo file.
2130         continue;
2131       }
2132 
2133       if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2134         // do not copy the .oldlogs files
2135         continue;
2136       }
2137 
2138       LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2139       // FileSystem.rename is inconsistent with directories -- if the
2140       // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2141       // it moves the src into the dst dir resulting in (foo/a/b).  If
2142       // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2143       for (FileStatus hfile : fs.listStatus(src)) {
2144         boolean success = fs.rename(hfile.getPath(), dst);
2145         if (success) {
2146           fileMoves++;
2147         }
2148       }
2149       LOG.debug("[" + thread + "] Sideline directory contents:");
2150       debugLsr(targetRegionDir);
2151     }
2152 
2153     // if all success.
2154     sidelineRegionDir(fs, contained);
2155     LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2156         getSidelineDir());
2157     debugLsr(contained.getHdfsRegionDir());
2158 
2159     return fileMoves;
2160   }
2161 
2162 
2163   static class WorkItemOverlapMerge implements Callable<Void> {
2164     private TableIntegrityErrorHandler handler;
2165     Collection<HbckInfo> overlapgroup;
2166 
2167     WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2168       this.handler = handler;
2169       this.overlapgroup = overlapgroup;
2170     }
2171 
2172     @Override
2173     public Void call() throws Exception {
2174       handler.handleOverlapGroup(overlapgroup);
2175       return null;
2176     }
2177   };
2178 
2179 
2180   /**
2181    * Maintain information about a particular table.
2182    */
2183   public class TableInfo {
2184     TableName tableName;
2185     TreeSet <ServerName> deployedOn;
2186 
2187     // backwards regions
2188     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2189 
2190     // sidelined big overlapped regions
2191     final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2192 
2193     // region split calculator
2194     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2195 
2196     // Histogram of different HTableDescriptors found.  Ideally there is only one!
2197     final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2198 
2199     // key = start split, values = set of splits in problem group
2200     final Multimap<byte[], HbckInfo> overlapGroups =
2201       TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2202 
2203     TableInfo(TableName name) {
2204       this.tableName = name;
2205       deployedOn = new TreeSet <ServerName>();
2206     }
2207 
2208     /**
2209      * @return descriptor common to all regions.  null if are none or multiple!
2210      */
2211     private HTableDescriptor getHTD() {
2212       if (htds.size() == 1) {
2213         return (HTableDescriptor)htds.toArray()[0];
2214       } else {
2215         LOG.error("None/Multiple table descriptors found for table '"
2216           + tableName + "' regions: " + htds);
2217       }
2218       return null;
2219     }
2220 
2221     public void addRegionInfo(HbckInfo hir) {
2222       if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2223         // end key is absolute end key, just add it.
2224         // ignore replicas other than primary for these checks
2225         if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2226         return;
2227       }
2228 
2229       // if not the absolute end key, check for cycle
2230       if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2231         errors.reportError(
2232             ERROR_CODE.REGION_CYCLE,
2233             String.format("The endkey for this region comes before the "
2234                 + "startkey, startkey=%s, endkey=%s",
2235                 Bytes.toStringBinary(hir.getStartKey()),
2236                 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2237         backwards.add(hir);
2238         return;
2239       }
2240 
2241       // main case, add to split calculator
2242       // ignore replicas other than primary for these checks
2243       if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2244     }
2245 
2246     public void addServer(ServerName server) {
2247       this.deployedOn.add(server);
2248     }
2249 
2250     public TableName getName() {
2251       return tableName;
2252     }
2253 
2254     public int getNumRegions() {
2255       return sc.getStarts().size() + backwards.size();
2256     }
2257 
2258     private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2259       ErrorReporter errors;
2260 
2261       IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2262         this.errors = errors;
2263         setTableInfo(ti);
2264       }
2265 
2266       @Override
2267       public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2268         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2269             "First region should start with an empty key.  You need to "
2270             + " create a new region and regioninfo in HDFS to plug the hole.",
2271             getTableInfo(), hi);
2272       }
2273 
2274       @Override
2275       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2276         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2277             "Last region should end with an empty key. You need to "
2278                 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2279       }
2280 
2281       @Override
2282       public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2283         errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2284             "Region has the same start and end key.", getTableInfo(), hi);
2285       }
2286 
2287       @Override
2288       public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2289         byte[] key = r1.getStartKey();
2290         // dup start key
2291         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2292             "Multiple regions have the same startkey: "
2293             + Bytes.toStringBinary(key), getTableInfo(), r1);
2294         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2295             "Multiple regions have the same startkey: "
2296             + Bytes.toStringBinary(key), getTableInfo(), r2);
2297       }
2298 
2299       @Override
2300       public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2301         errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2302             "There is an overlap in the region chain.",
2303             getTableInfo(), hi1, hi2);
2304       }
2305 
2306       @Override
2307       public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2308         errors.reportError(
2309             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2310             "There is a hole in the region chain between "
2311                 + Bytes.toStringBinary(holeStart) + " and "
2312                 + Bytes.toStringBinary(holeStop)
2313                 + ".  You need to create a new .regioninfo and region "
2314                 + "dir in hdfs to plug the hole.");
2315       }
2316     };
2317 
2318     /**
2319      * This handler fixes integrity errors from hdfs information.  There are
2320      * basically three classes of integrity problems 1) holes, 2) overlaps, and
2321      * 3) invalid regions.
2322      *
2323      * This class overrides methods that fix holes and the overlap group case.
2324      * Individual cases of particular overlaps are handled by the general
2325      * overlap group merge repair case.
2326      *
2327      * If hbase is online, this forces regions offline before doing merge
2328      * operations.
2329      */
2330     private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2331       Configuration conf;
2332 
2333       boolean fixOverlaps = true;
2334 
2335       HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2336           boolean fixHoles, boolean fixOverlaps) {
2337         super(ti, errors);
2338         this.conf = conf;
2339         this.fixOverlaps = fixOverlaps;
2340         // TODO properly use fixHoles
2341       }
2342 
2343       /**
2344        * This is a special case hole -- when the first region of a table is
2345        * missing from META, HBase doesn't acknowledge the existance of the
2346        * table.
2347        */
2348       @Override
2349       public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2350         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2351             "First region should start with an empty key.  Creating a new " +
2352             "region and regioninfo in HDFS to plug the hole.",
2353             getTableInfo(), next);
2354         HTableDescriptor htd = getTableInfo().getHTD();
2355         // from special EMPTY_START_ROW to next region's startKey
2356         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2357             HConstants.EMPTY_START_ROW, next.getStartKey());
2358 
2359         // TODO test
2360         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2361         LOG.info("Table region start key was not empty.  Created new empty region: "
2362             + newRegion + " " +region);
2363         fixes++;
2364       }
2365 
2366       @Override
2367       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2368         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2369             "Last region should end with an empty key.  Creating a new "
2370                 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2371         HTableDescriptor htd = getTableInfo().getHTD();
2372         // from curEndKey to EMPTY_START_ROW
2373         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2374             HConstants.EMPTY_START_ROW);
2375 
2376         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2377         LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
2378             + " " + region);
2379         fixes++;
2380       }
2381 
2382       /**
2383        * There is a hole in the hdfs regions that violates the table integrity
2384        * rules.  Create a new empty region that patches the hole.
2385        */
2386       @Override
2387       public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2388         errors.reportError(
2389             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2390             "There is a hole in the region chain between "
2391                 + Bytes.toStringBinary(holeStartKey) + " and "
2392                 + Bytes.toStringBinary(holeStopKey)
2393                 + ".  Creating a new regioninfo and region "
2394                 + "dir in hdfs to plug the hole.");
2395         HTableDescriptor htd = getTableInfo().getHTD();
2396         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2397         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2398         LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2399         fixes++;
2400       }
2401 
2402       /**
2403        * This takes set of overlapping regions and merges them into a single
2404        * region.  This covers cases like degenerate regions, shared start key,
2405        * general overlaps, duplicate ranges, and partial overlapping regions.
2406        *
2407        * Cases:
2408        * - Clean regions that overlap
2409        * - Only .oldlogs regions (can't find start/stop range, or figure out)
2410        *
2411        * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
2412        */
2413       @Override
2414       public void handleOverlapGroup(Collection<HbckInfo> overlap)
2415           throws IOException {
2416         Preconditions.checkNotNull(overlap);
2417         Preconditions.checkArgument(overlap.size() >0);
2418 
2419         if (!this.fixOverlaps) {
2420           LOG.warn("Not attempting to repair overlaps.");
2421           return;
2422         }
2423 
2424         if (overlap.size() > maxMerge) {
2425           LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2426             "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2427           if (sidelineBigOverlaps) {
2428             // we only sideline big overlapped groups that exceeds the max number of regions to merge
2429             sidelineBigOverlaps(overlap);
2430           }
2431           return;
2432         }
2433 
2434         mergeOverlaps(overlap);
2435       }
2436 
2437       void mergeOverlaps(Collection<HbckInfo> overlap)
2438           throws IOException {
2439         String thread = Thread.currentThread().getName();
2440         LOG.info("== [" + thread + "] Merging regions into one region: "
2441           + Joiner.on(",").join(overlap));
2442         // get the min / max range and close all concerned regions
2443         Pair<byte[], byte[]> range = null;
2444         for (HbckInfo hi : overlap) {
2445           if (range == null) {
2446             range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2447           } else {
2448             if (RegionSplitCalculator.BYTES_COMPARATOR
2449                 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2450               range.setFirst(hi.getStartKey());
2451             }
2452             if (RegionSplitCalculator.BYTES_COMPARATOR
2453                 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2454               range.setSecond(hi.getEndKey());
2455             }
2456           }
2457           // need to close files so delete can happen.
2458           LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
2459           LOG.debug("[" + thread + "] Contained region dir before close");
2460           debugLsr(hi.getHdfsRegionDir());
2461           try {
2462             LOG.info("[" + thread + "] Closing region: " + hi);
2463             closeRegion(hi);
2464           } catch (IOException ioe) {
2465             LOG.warn("[" + thread + "] Was unable to close region " + hi
2466               + ".  Just continuing... ", ioe);
2467           } catch (InterruptedException e) {
2468             LOG.warn("[" + thread + "] Was unable to close region " + hi
2469               + ".  Just continuing... ", e);
2470           }
2471 
2472           try {
2473             LOG.info("[" + thread + "] Offlining region: " + hi);
2474             offline(hi.getRegionName());
2475           } catch (IOException ioe) {
2476             LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2477               + ".  Just continuing... ", ioe);
2478           }
2479         }
2480 
2481         // create new empty container region.
2482         HTableDescriptor htd = getTableInfo().getHTD();
2483         // from start key to end Key
2484         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
2485             range.getSecond());
2486         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2487         LOG.info("[" + thread + "] Created new empty container region: " +
2488             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
2489         debugLsr(region.getRegionFileSystem().getRegionDir());
2490 
2491         // all target regions are closed, should be able to safely cleanup.
2492         boolean didFix= false;
2493         Path target = region.getRegionFileSystem().getRegionDir();
2494         for (HbckInfo contained : overlap) {
2495           LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
2496           int merges = mergeRegionDirs(target, contained);
2497           if (merges > 0) {
2498             didFix = true;
2499           }
2500         }
2501         if (didFix) {
2502           fixes++;
2503         }
2504       }
2505 
2506       /**
2507        * Sideline some regions in a big overlap group so that it
2508        * will have fewer regions, and it is easier to merge them later on.
2509        *
2510        * @param bigOverlap the overlapped group with regions more than maxMerge
2511        * @throws IOException
2512        */
2513       void sidelineBigOverlaps(
2514           Collection<HbckInfo> bigOverlap) throws IOException {
2515         int overlapsToSideline = bigOverlap.size() - maxMerge;
2516         if (overlapsToSideline > maxOverlapsToSideline) {
2517           overlapsToSideline = maxOverlapsToSideline;
2518         }
2519         List<HbckInfo> regionsToSideline =
2520           RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
2521         FileSystem fs = FileSystem.get(conf);
2522         for (HbckInfo regionToSideline: regionsToSideline) {
2523           try {
2524             LOG.info("Closing region: " + regionToSideline);
2525             closeRegion(regionToSideline);
2526           } catch (IOException ioe) {
2527             LOG.warn("Was unable to close region " + regionToSideline
2528               + ".  Just continuing... ", ioe);
2529           } catch (InterruptedException e) {
2530             LOG.warn("Was unable to close region " + regionToSideline
2531               + ".  Just continuing... ", e);
2532           }
2533 
2534           try {
2535             LOG.info("Offlining region: " + regionToSideline);
2536             offline(regionToSideline.getRegionName());
2537           } catch (IOException ioe) {
2538             LOG.warn("Unable to offline region from master: " + regionToSideline
2539               + ".  Just continuing... ", ioe);
2540           }
2541 
2542           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
2543           Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
2544           if (sidelineRegionDir != null) {
2545             sidelinedRegions.put(sidelineRegionDir, regionToSideline);
2546             LOG.info("After sidelined big overlapped region: "
2547               + regionToSideline.getRegionNameAsString()
2548               + " to " + sidelineRegionDir.toString());
2549             fixes++;
2550           }
2551         }
2552       }
2553     }
2554 
2555     /**
2556      * Check the region chain (from META) of this table.  We are looking for
2557      * holes, overlaps, and cycles.
2558      * @return false if there are errors
2559      * @throws IOException
2560      */
2561     public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
2562       // When table is disabled no need to check for the region chain. Some of the regions
2563       // accidently if deployed, this below code might report some issues like missing start
2564       // or end regions or region hole in chain and may try to fix which is unwanted.
2565       if (disabledTables.contains(this.tableName)) {
2566         return true;
2567       }
2568       int originalErrorsCount = errors.getErrorList().size();
2569       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
2570       SortedSet<byte[]> splits = sc.getSplits();
2571 
2572       byte[] prevKey = null;
2573       byte[] problemKey = null;
2574 
2575       if (splits.size() == 0) {
2576         // no region for this table
2577         handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
2578       }
2579 
2580       for (byte[] key : splits) {
2581         Collection<HbckInfo> ranges = regions.get(key);
2582         if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
2583           for (HbckInfo rng : ranges) {
2584             handler.handleRegionStartKeyNotEmpty(rng);
2585           }
2586         }
2587 
2588         // check for degenerate ranges
2589         for (HbckInfo rng : ranges) {
2590           // special endkey case converts '' to null
2591           byte[] endKey = rng.getEndKey();
2592           endKey = (endKey.length == 0) ? null : endKey;
2593           if (Bytes.equals(rng.getStartKey(),endKey)) {
2594             handler.handleDegenerateRegion(rng);
2595           }
2596         }
2597 
2598         if (ranges.size() == 1) {
2599           // this split key is ok -- no overlap, not a hole.
2600           if (problemKey != null) {
2601             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2602           }
2603           problemKey = null; // fell through, no more problem.
2604         } else if (ranges.size() > 1) {
2605           // set the new problem key group name, if already have problem key, just
2606           // keep using it.
2607           if (problemKey == null) {
2608             // only for overlap regions.
2609             LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
2610             problemKey = key;
2611           }
2612           overlapGroups.putAll(problemKey, ranges);
2613 
2614           // record errors
2615           ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
2616           //  this dumb and n^2 but this shouldn't happen often
2617           for (HbckInfo r1 : ranges) {
2618             if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2619             subRange.remove(r1);
2620             for (HbckInfo r2 : subRange) {
2621               if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2622               if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
2623                 handler.handleDuplicateStartKeys(r1,r2);
2624               } else {
2625                 // overlap
2626                 handler.handleOverlapInRegionChain(r1, r2);
2627               }
2628             }
2629           }
2630 
2631         } else if (ranges.size() == 0) {
2632           if (problemKey != null) {
2633             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2634           }
2635           problemKey = null;
2636 
2637           byte[] holeStopKey = sc.getSplits().higher(key);
2638           // if higher key is null we reached the top.
2639           if (holeStopKey != null) {
2640             // hole
2641             handler.handleHoleInRegionChain(key, holeStopKey);
2642           }
2643         }
2644         prevKey = key;
2645       }
2646 
2647       // When the last region of a table is proper and having an empty end key, 'prevKey'
2648       // will be null.
2649       if (prevKey != null) {
2650         handler.handleRegionEndKeyNotEmpty(prevKey);
2651       }
2652 
2653       // TODO fold this into the TableIntegrityHandler
2654       if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
2655         LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
2656             " false to run serially.");
2657         boolean ok = handleOverlapsParallel(handler, prevKey);
2658         if (!ok) {
2659           return false;
2660         }
2661       } else {
2662         LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
2663             " true to run in parallel.");
2664         for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2665           handler.handleOverlapGroup(overlap);
2666         }
2667       }
2668 
2669       if (details) {
2670         // do full region split map dump
2671         errors.print("---- Table '"  +  this.tableName
2672             + "': region split map");
2673         dump(splits, regions);
2674         errors.print("---- Table '"  +  this.tableName
2675             + "': overlap groups");
2676         dumpOverlapProblems(overlapGroups);
2677         errors.print("There are " + overlapGroups.keySet().size()
2678             + " overlap groups with " + overlapGroups.size()
2679             + " overlapping regions");
2680       }
2681       if (!sidelinedRegions.isEmpty()) {
2682         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
2683         errors.print("---- Table '"  +  this.tableName
2684             + "': sidelined big overlapped regions");
2685         dumpSidelinedRegions(sidelinedRegions);
2686       }
2687       return errors.getErrorList().size() == originalErrorsCount;
2688     }
2689 
2690     private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
2691         throws IOException {
2692       // we parallelize overlap handler for the case we have lots of groups to fix.  We can
2693       // safely assume each group is independent.
2694       List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
2695       List<Future<Void>> rets;
2696       for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2697         //
2698         merges.add(new WorkItemOverlapMerge(overlap, handler));
2699       }
2700       try {
2701         rets = executor.invokeAll(merges);
2702       } catch (InterruptedException e) {
2703         LOG.error("Overlap merges were interrupted", e);
2704         return false;
2705       }
2706       for(int i=0; i<merges.size(); i++) {
2707         WorkItemOverlapMerge work = merges.get(i);
2708         Future<Void> f = rets.get(i);
2709         try {
2710           f.get();
2711         } catch(ExecutionException e) {
2712           LOG.warn("Failed to merge overlap group" + work, e.getCause());
2713         } catch (InterruptedException e) {
2714           LOG.error("Waiting for overlap merges was interrupted", e);
2715           return false;
2716         }
2717       }
2718       return true;
2719     }
2720 
2721     /**
2722      * This dumps data in a visually reasonable way for visual debugging
2723      *
2724      * @param splits
2725      * @param regions
2726      */
2727     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
2728       // we display this way because the last end key should be displayed as well.
2729       StringBuilder sb = new StringBuilder();
2730       for (byte[] k : splits) {
2731         sb.setLength(0); // clear out existing buffer, if any.
2732         sb.append(Bytes.toStringBinary(k) + ":\t");
2733         for (HbckInfo r : regions.get(k)) {
2734           sb.append("[ "+ r.toString() + ", "
2735               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
2736         }
2737         errors.print(sb.toString());
2738       }
2739     }
2740   }
2741 
2742   public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
2743     // we display this way because the last end key should be displayed as
2744     // well.
2745     for (byte[] k : regions.keySet()) {
2746       errors.print(Bytes.toStringBinary(k) + ":");
2747       for (HbckInfo r : regions.get(k)) {
2748         errors.print("[ " + r.toString() + ", "
2749             + Bytes.toStringBinary(r.getEndKey()) + "]");
2750       }
2751       errors.print("----");
2752     }
2753   }
2754 
2755   public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
2756     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
2757       TableName tableName = entry.getValue().getTableName();
2758       Path path = entry.getKey();
2759       errors.print("This sidelined region dir should be bulk loaded: "
2760         + path.toString());
2761       errors.print("Bulk load command looks like: "
2762         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
2763         + path.toUri().getPath() + " "+ tableName);
2764     }
2765   }
2766 
2767   public Multimap<byte[], HbckInfo> getOverlapGroups(
2768       TableName table) {
2769     TableInfo ti = tablesInfo.get(table);
2770     return ti.overlapGroups;
2771   }
2772 
2773   /**
2774    * Return a list of user-space table names whose metadata have not been
2775    * modified in the last few milliseconds specified by timelag
2776    * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
2777    * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
2778    * milliseconds specified by timelag, then the table is a candidate to be returned.
2779    * @return tables that have not been modified recently
2780    * @throws IOException if an error is encountered
2781    */
2782   HTableDescriptor[] getTables(AtomicInteger numSkipped) {
2783     List<TableName> tableNames = new ArrayList<TableName>();
2784     long now = System.currentTimeMillis();
2785 
2786     for (HbckInfo hbi : regionInfoMap.values()) {
2787       MetaEntry info = hbi.metaEntry;
2788 
2789       // if the start key is zero, then we have found the first region of a table.
2790       // pick only those tables that were not modified in the last few milliseconds.
2791       if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
2792         if (info.modTime + timelag < now) {
2793           tableNames.add(info.getTable());
2794         } else {
2795           numSkipped.incrementAndGet(); // one more in-flux table
2796         }
2797       }
2798     }
2799     return getHTableDescriptors(tableNames);
2800   }
2801 
2802   HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
2803     HTableDescriptor[] htd = new HTableDescriptor[0];
2804     Admin admin = null;
2805     try {
2806       LOG.info("getHTableDescriptors == tableNames => " + tableNames);
2807       admin = new HBaseAdmin(getConf());
2808       htd = admin.getTableDescriptorsByTableName(tableNames);
2809     } catch (IOException e) {
2810       LOG.debug("Exception getting table descriptors", e);
2811     } finally {
2812       if (admin != null) {
2813         try {
2814           admin.close();
2815         } catch (IOException e) {
2816           LOG.debug("Exception closing HBaseAdmin", e);
2817         }
2818       }
2819     }
2820     return htd;
2821   }
2822 
2823   /**
2824    * Gets the entry in regionInfo corresponding to the the given encoded
2825    * region name. If the region has not been seen yet, a new entry is added
2826    * and returned.
2827    */
2828   private synchronized HbckInfo getOrCreateInfo(String name) {
2829     HbckInfo hbi = regionInfoMap.get(name);
2830     if (hbi == null) {
2831       hbi = new HbckInfo(null);
2832       regionInfoMap.put(name, hbi);
2833     }
2834     return hbi;
2835   }
2836 
2837   private void checkAndFixTableLocks() throws IOException {
2838     TableLockChecker checker = new TableLockChecker(createZooKeeperWatcher(), errors);
2839     checker.checkTableLocks();
2840 
2841     if (this.fixTableLocks) {
2842       checker.fixExpiredTableLocks();
2843     }
2844   }
2845 
2846   /**
2847     * Check values in regionInfo for hbase:meta
2848     * Check if zero or more than one regions with hbase:meta are found.
2849     * If there are inconsistencies (i.e. zero or more than one regions
2850     * pretend to be holding the hbase:meta) try to fix that and report an error.
2851     * @throws IOException from HBaseFsckRepair functions
2852    * @throws KeeperException
2853    * @throws InterruptedException
2854     */
2855   boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
2856     List<HbckInfo> metaRegions = Lists.newArrayList();
2857     for (HbckInfo value : regionInfoMap.values()) {
2858       if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
2859         metaRegions.add(value);
2860       }
2861     }
2862 
2863     // There will be always one entry in regionInfoMap corresponding to hbase:meta
2864     // Check the deployed servers. It should be exactly one server.
2865     List<ServerName> servers = new ArrayList<ServerName>();
2866     HbckInfo metaHbckInfo = null;
2867     if (!metaRegions.isEmpty()) {
2868       metaHbckInfo = metaRegions.get(0);
2869       servers = metaHbckInfo.deployedOn;
2870     }
2871     if (servers.size() != 1) {
2872       if (servers.size() == 0) {
2873         errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta is not found on any region.");
2874         if (shouldFixAssignments()) {
2875           errors.print("Trying to fix a problem with hbase:meta..");
2876           setShouldRerun();
2877           // try to fix it (treat it as unassigned region)
2878           HBaseFsckRepair.fixUnassigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
2879           HBaseFsckRepair.waitUntilAssigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
2880         }
2881       } else if (servers.size() > 1) {
2882         errors
2883             .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta is found on more than one region.");
2884         if (shouldFixAssignments()) {
2885           if (metaHbckInfo == null) {
2886             errors.print(
2887               "Unable to fix problem with hbase:meta due to hbase:meta region info missing");
2888             return false;
2889           }
2890           errors.print("Trying to fix a problem with hbase:meta..");
2891           setShouldRerun();
2892           // try fix it (treat is a dupe assignment)
2893           HBaseFsckRepair.fixMultiAssignment(admin, metaHbckInfo.metaEntry, servers);
2894         }
2895       }
2896       // rerun hbck with hopefully fixed META
2897       return false;
2898     }
2899     // no errors, so continue normally
2900     return true;
2901   }
2902 
2903   /**
2904    * Scan hbase:meta, adding all regions found to the regionInfo map.
2905    * @throws IOException if an error is encountered
2906    */
2907   boolean loadMetaEntries() throws IOException {
2908     MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
2909       int countRecord = 1;
2910 
2911       // comparator to sort KeyValues with latest modtime
2912       final Comparator<Cell> comp = new Comparator<Cell>() {
2913         @Override
2914         public int compare(Cell k1, Cell k2) {
2915           return (int)(k1.getTimestamp() - k2.getTimestamp());
2916         }
2917       };
2918 
2919       @Override
2920       public boolean processRow(Result result) throws IOException {
2921         try {
2922 
2923           // record the latest modification of this META record
2924           long ts =  Collections.max(result.listCells(), comp).getTimestamp();
2925           RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
2926           if (rl == null) {
2927             emptyRegionInfoQualifiers.add(result);
2928             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
2929               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
2930             return true;
2931           }
2932           ServerName sn = null;
2933           if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
2934               rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
2935             emptyRegionInfoQualifiers.add(result);
2936             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
2937               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
2938             return true;
2939           }
2940           HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
2941           if (!(isTableIncluded(hri.getTable())
2942               || hri.isMetaRegion())) {
2943             return true;
2944           }
2945           PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
2946           for (HRegionLocation h : rl.getRegionLocations()) {
2947             if (h == null || h.getRegionInfo() == null) {
2948               continue;
2949             }
2950             sn = h.getServerName();
2951             hri = h.getRegionInfo();
2952 
2953             MetaEntry m = null;
2954             if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2955               m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
2956             } else {
2957               m = new MetaEntry(hri, sn, ts, null, null);
2958             }
2959             HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
2960             if (previous == null) {
2961               regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
2962             } else if (previous.metaEntry == null) {
2963               previous.metaEntry = m;
2964             } else {
2965               throw new IOException("Two entries in hbase:meta are same " + previous);
2966             }
2967           }
2968           PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
2969           for (HRegionInfo mergeRegion : new HRegionInfo[] {
2970               mergeRegions.getFirst(), mergeRegions.getSecond() }) {
2971             if (mergeRegion != null) {
2972               // This region is already been merged
2973               HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
2974               hbInfo.setMerged(true);
2975             }
2976           }
2977 
2978           // show proof of progress to the user, once for every 100 records.
2979           if (countRecord % 100 == 0) {
2980             errors.progress();
2981           }
2982           countRecord++;
2983           return true;
2984         } catch (RuntimeException e) {
2985           LOG.error("Result=" + result);
2986           throw e;
2987         }
2988       }
2989     };
2990     if (!checkMetaOnly) {
2991       // Scan hbase:meta to pick up user regions
2992       MetaScanner.metaScan(getConf(), visitor);
2993     }
2994 
2995     errors.print("");
2996     return true;
2997   }
2998 
2999   /**
3000    * Stores the regioninfo entries scanned from META
3001    */
3002   static class MetaEntry extends HRegionInfo {
3003     ServerName regionServer;   // server hosting this region
3004     long modTime;          // timestamp of most recent modification metadata
3005     HRegionInfo splitA, splitB; //split daughters
3006 
3007     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3008       this(rinfo, regionServer, modTime, null, null);
3009     }
3010 
3011     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3012         HRegionInfo splitA, HRegionInfo splitB) {
3013       super(rinfo);
3014       this.regionServer = regionServer;
3015       this.modTime = modTime;
3016       this.splitA = splitA;
3017       this.splitB = splitB;
3018     }
3019 
3020     @Override
3021     public boolean equals(Object o) {
3022       boolean superEq = super.equals(o);
3023       if (!superEq) {
3024         return superEq;
3025       }
3026 
3027       MetaEntry me = (MetaEntry) o;
3028       if (!regionServer.equals(me.regionServer)) {
3029         return false;
3030       }
3031       return (modTime == me.modTime);
3032     }
3033 
3034     @Override
3035     public int hashCode() {
3036       int hash = Arrays.hashCode(getRegionName());
3037       hash ^= getRegionId();
3038       hash ^= Arrays.hashCode(getStartKey());
3039       hash ^= Arrays.hashCode(getEndKey());
3040       hash ^= Boolean.valueOf(isOffline()).hashCode();
3041       hash ^= getTable().hashCode();
3042       if (regionServer != null) {
3043         hash ^= regionServer.hashCode();
3044       }
3045       hash ^= modTime;
3046       return hash;
3047     }
3048   }
3049 
3050   /**
3051    * Stores the regioninfo entries from HDFS
3052    */
3053   static class HdfsEntry {
3054     HRegionInfo hri;
3055     Path hdfsRegionDir = null;
3056     long hdfsRegionDirModTime  = 0;
3057     boolean hdfsRegioninfoFilePresent = false;
3058     boolean hdfsOnlyEdits = false;
3059   }
3060 
3061   /**
3062    * Stores the regioninfo retrieved from Online region servers.
3063    */
3064   static class OnlineEntry {
3065     HRegionInfo hri;
3066     ServerName hsa;
3067 
3068     @Override
3069     public String toString() {
3070       return hsa.toString() + ";" + hri.getRegionNameAsString();
3071     }
3072   }
3073 
3074   /**
3075    * Maintain information about a particular region.  It gathers information
3076    * from three places -- HDFS, META, and region servers.
3077    */
3078   public static class HbckInfo implements KeyRange {
3079     private MetaEntry metaEntry = null; // info in META
3080     private HdfsEntry hdfsEntry = null; // info in HDFS
3081     private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
3082     private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
3083     private boolean skipChecks = false; // whether to skip further checks to this region info.
3084     private boolean isMerged = false;// whether this region has already been merged into another one
3085     private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3086     private HRegionInfo primaryHRIForDeployedReplica = null;
3087 
3088     HbckInfo(MetaEntry metaEntry) {
3089       this.metaEntry = metaEntry;
3090     }
3091 
3092     public int getReplicaId() {
3093       if (metaEntry != null) return metaEntry.getReplicaId();
3094       return deployedReplicaId;
3095     }
3096 
3097     public synchronized void addServer(HRegionInfo hri, ServerName server) {
3098       OnlineEntry rse = new OnlineEntry() ;
3099       rse.hri = hri;
3100       rse.hsa = server;
3101       this.deployedEntries.add(rse);
3102       this.deployedOn.add(server);
3103       // save the replicaId that we see deployed in the cluster
3104       this.deployedReplicaId = hri.getReplicaId();
3105       this.primaryHRIForDeployedReplica =
3106           RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3107     }
3108 
3109     @Override
3110     public synchronized String toString() {
3111       StringBuilder sb = new StringBuilder();
3112       sb.append("{ meta => ");
3113       sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3114       sb.append( ", hdfs => " + getHdfsRegionDir());
3115       sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3116       sb.append( ", replicaId => " + getReplicaId());
3117       sb.append(" }");
3118       return sb.toString();
3119     }
3120 
3121     @Override
3122     public byte[] getStartKey() {
3123       if (this.metaEntry != null) {
3124         return this.metaEntry.getStartKey();
3125       } else if (this.hdfsEntry != null) {
3126         return this.hdfsEntry.hri.getStartKey();
3127       } else {
3128         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3129         return null;
3130       }
3131     }
3132 
3133     @Override
3134     public byte[] getEndKey() {
3135       if (this.metaEntry != null) {
3136         return this.metaEntry.getEndKey();
3137       } else if (this.hdfsEntry != null) {
3138         return this.hdfsEntry.hri.getEndKey();
3139       } else {
3140         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3141         return null;
3142       }
3143     }
3144 
3145     public TableName getTableName() {
3146       if (this.metaEntry != null) {
3147         return this.metaEntry.getTable();
3148       } else if (this.hdfsEntry != null) {
3149         // we are only guaranteed to have a path and not an HRI for hdfsEntry,
3150         // so we get the name from the Path
3151         Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3152         return FSUtils.getTableName(tableDir);
3153       } else {
3154         // return the info from the first online/deployed hri
3155         for (OnlineEntry e : deployedEntries) {
3156           return e.hri.getTable();
3157         }
3158         return null;
3159       }
3160     }
3161 
3162     public String getRegionNameAsString() {
3163       if (metaEntry != null) {
3164         return metaEntry.getRegionNameAsString();
3165       } else if (hdfsEntry != null) {
3166         if (hdfsEntry.hri != null) {
3167           return hdfsEntry.hri.getRegionNameAsString();
3168         }
3169       } else {
3170         // return the info from the first online/deployed hri
3171         for (OnlineEntry e : deployedEntries) {
3172           return e.hri.getRegionNameAsString();
3173         }
3174       }
3175       return null;
3176     }
3177 
3178     public byte[] getRegionName() {
3179       if (metaEntry != null) {
3180         return metaEntry.getRegionName();
3181       } else if (hdfsEntry != null) {
3182         return hdfsEntry.hri.getRegionName();
3183       } else {
3184         // return the info from the first online/deployed hri
3185         for (OnlineEntry e : deployedEntries) {
3186           return e.hri.getRegionName();
3187         }
3188         return null;
3189       }
3190     }
3191 
3192     public HRegionInfo getPrimaryHRIForDeployedReplica() {
3193       return primaryHRIForDeployedReplica;
3194     }
3195 
3196     Path getHdfsRegionDir() {
3197       if (hdfsEntry == null) {
3198         return null;
3199       }
3200       return hdfsEntry.hdfsRegionDir;
3201     }
3202 
3203     boolean containsOnlyHdfsEdits() {
3204       if (hdfsEntry == null) {
3205         return false;
3206       }
3207       return hdfsEntry.hdfsOnlyEdits;
3208     }
3209 
3210     boolean isHdfsRegioninfoPresent() {
3211       if (hdfsEntry == null) {
3212         return false;
3213       }
3214       return hdfsEntry.hdfsRegioninfoFilePresent;
3215     }
3216 
3217     long getModTime() {
3218       if (hdfsEntry == null) {
3219         return 0;
3220       }
3221       return hdfsEntry.hdfsRegionDirModTime;
3222     }
3223 
3224     HRegionInfo getHdfsHRI() {
3225       if (hdfsEntry == null) {
3226         return null;
3227       }
3228       return hdfsEntry.hri;
3229     }
3230 
3231     public void setSkipChecks(boolean skipChecks) {
3232       this.skipChecks = skipChecks;
3233     }
3234 
3235     public boolean isSkipChecks() {
3236       return skipChecks;
3237     }
3238 
3239     public void setMerged(boolean isMerged) {
3240       this.isMerged = isMerged;
3241     }
3242 
3243     public boolean isMerged() {
3244       return this.isMerged;
3245     }
3246   }
3247 
3248   final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3249     @Override
3250     public int compare(HbckInfo l, HbckInfo r) {
3251       if (l == r) {
3252         // same instance
3253         return 0;
3254       }
3255 
3256       int tableCompare = l.getTableName().compareTo(r.getTableName());
3257       if (tableCompare != 0) {
3258         return tableCompare;
3259       }
3260 
3261       int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3262           l.getStartKey(), r.getStartKey());
3263       if (startComparison != 0) {
3264         return startComparison;
3265       }
3266 
3267       // Special case for absolute endkey
3268       byte[] endKey = r.getEndKey();
3269       endKey = (endKey.length == 0) ? null : endKey;
3270       byte[] endKey2 = l.getEndKey();
3271       endKey2 = (endKey2.length == 0) ? null : endKey2;
3272       int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3273           endKey2,  endKey);
3274 
3275       if (endComparison != 0) {
3276         return endComparison;
3277       }
3278 
3279       // use regionId as tiebreaker.
3280       // Null is considered after all possible values so make it bigger.
3281       if (l.hdfsEntry == null && r.hdfsEntry == null) {
3282         return 0;
3283       }
3284       if (l.hdfsEntry == null && r.hdfsEntry != null) {
3285         return 1;
3286       }
3287       // l.hdfsEntry must not be null
3288       if (r.hdfsEntry == null) {
3289         return -1;
3290       }
3291       // both l.hdfsEntry and r.hdfsEntry must not be null.
3292       return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3293     }
3294   };
3295 
3296   /**
3297    * Prints summary of all tables found on the system.
3298    */
3299   private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3300     StringBuilder sb = new StringBuilder();
3301     errors.print("Summary:");
3302     for (TableInfo tInfo : tablesInfo.values()) {
3303       if (errors.tableHasErrors(tInfo)) {
3304         errors.print("Table " + tInfo.getName() + " is inconsistent.");
3305       } else {
3306         errors.print("  " + tInfo.getName() + " is okay.");
3307       }
3308       errors.print("    Number of regions: " + tInfo.getNumRegions());
3309       sb.setLength(0); // clear out existing buffer, if any.
3310       sb.append("    Deployed on: ");
3311       for (ServerName server : tInfo.deployedOn) {
3312         sb.append(" " + server.toString());
3313       }
3314       errors.print(sb.toString());
3315     }
3316   }
3317 
3318   static ErrorReporter getErrorReporter(
3319       final Configuration conf) throws ClassNotFoundException {
3320     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3321     return (ErrorReporter)ReflectionUtils.newInstance(reporter, conf);
3322   }
3323 
3324   public interface ErrorReporter {
3325     enum ERROR_CODE {
3326       UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3327       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
3328       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3329       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3330       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3331       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3332       WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR
3333     }
3334     void clear();
3335     void report(String message);
3336     void reportError(String message);
3337     void reportError(ERROR_CODE errorCode, String message);
3338     void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3339     void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3340     void reportError(
3341       ERROR_CODE errorCode,
3342       String message,
3343       TableInfo table,
3344       HbckInfo info1,
3345       HbckInfo info2
3346     );
3347     int summarize();
3348     void detail(String details);
3349     ArrayList<ERROR_CODE> getErrorList();
3350     void progress();
3351     void print(String message);
3352     void resetErrors();
3353     boolean tableHasErrors(TableInfo table);
3354   }
3355 
3356   static class PrintingErrorReporter implements ErrorReporter {
3357     public int errorCount = 0;
3358     private int showProgress;
3359 
3360     Set<TableInfo> errorTables = new HashSet<TableInfo>();
3361 
3362     // for use by unit tests to verify which errors were discovered
3363     private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3364 
3365     @Override
3366     public void clear() {
3367       errorTables.clear();
3368       errorList.clear();
3369       errorCount = 0;
3370     }
3371 
3372     @Override
3373     public synchronized void reportError(ERROR_CODE errorCode, String message) {
3374       if (errorCode == ERROR_CODE.WRONG_USAGE) {
3375         System.err.println(message);
3376         return;
3377       }
3378 
3379       errorList.add(errorCode);
3380       if (!summary) {
3381         System.out.println("ERROR: " + message);
3382       }
3383       errorCount++;
3384       showProgress = 0;
3385     }
3386 
3387     @Override
3388     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
3389       errorTables.add(table);
3390       reportError(errorCode, message);
3391     }
3392 
3393     @Override
3394     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3395                                          HbckInfo info) {
3396       errorTables.add(table);
3397       String reference = "(region " + info.getRegionNameAsString() + ")";
3398       reportError(errorCode, reference + " " + message);
3399     }
3400 
3401     @Override
3402     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3403                                          HbckInfo info1, HbckInfo info2) {
3404       errorTables.add(table);
3405       String reference = "(regions " + info1.getRegionNameAsString()
3406           + " and " + info2.getRegionNameAsString() + ")";
3407       reportError(errorCode, reference + " " + message);
3408     }
3409 
3410     @Override
3411     public synchronized void reportError(String message) {
3412       reportError(ERROR_CODE.UNKNOWN, message);
3413     }
3414 
3415     /**
3416      * Report error information, but do not increment the error count.  Intended for cases
3417      * where the actual error would have been reported previously.
3418      * @param message
3419      */
3420     @Override
3421     public synchronized void report(String message) {
3422       if (! summary) {
3423         System.out.println("ERROR: " + message);
3424       }
3425       showProgress = 0;
3426     }
3427 
3428     @Override
3429     public synchronized int summarize() {
3430       System.out.println(Integer.toString(errorCount) +
3431                          " inconsistencies detected.");
3432       if (errorCount == 0) {
3433         System.out.println("Status: OK");
3434         return 0;
3435       } else {
3436         System.out.println("Status: INCONSISTENT");
3437         return -1;
3438       }
3439     }
3440 
3441     @Override
3442     public ArrayList<ERROR_CODE> getErrorList() {
3443       return errorList;
3444     }
3445 
3446     @Override
3447     public synchronized void print(String message) {
3448       if (!summary) {
3449         System.out.println(message);
3450       }
3451     }
3452 
3453     @Override
3454     public boolean tableHasErrors(TableInfo table) {
3455       return errorTables.contains(table);
3456     }
3457 
3458     @Override
3459     public void resetErrors() {
3460       errorCount = 0;
3461     }
3462 
3463     @Override
3464     public synchronized void detail(String message) {
3465       if (details) {
3466         System.out.println(message);
3467       }
3468       showProgress = 0;
3469     }
3470 
3471     @Override
3472     public synchronized void progress() {
3473       if (showProgress++ == 10) {
3474         if (!summary) {
3475           System.out.print(".");
3476         }
3477         showProgress = 0;
3478       }
3479     }
3480   }
3481 
3482   /**
3483    * Contact a region server and get all information from it
3484    */
3485   static class WorkItemRegion implements Callable<Void> {
3486     private HBaseFsck hbck;
3487     private ServerName rsinfo;
3488     private ErrorReporter errors;
3489     private HConnection connection;
3490 
3491     WorkItemRegion(HBaseFsck hbck, ServerName info,
3492                    ErrorReporter errors, HConnection connection) {
3493       this.hbck = hbck;
3494       this.rsinfo = info;
3495       this.errors = errors;
3496       this.connection = connection;
3497     }
3498 
3499     @Override
3500     public synchronized Void call() throws IOException {
3501       errors.progress();
3502       try {
3503         BlockingInterface server = connection.getAdmin(rsinfo);
3504 
3505         // list all online regions from this region server
3506         List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
3507         regions = filterRegions(regions);
3508 
3509         if (details) {
3510           errors.detail("RegionServer: " + rsinfo.getServerName() +
3511                            " number of regions: " + regions.size());
3512           for (HRegionInfo rinfo: regions) {
3513             errors.detail("  " + rinfo.getRegionNameAsString() +
3514                              " id: " + rinfo.getRegionId() +
3515                              " encoded_name: " + rinfo.getEncodedName() +
3516                              " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
3517                              " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
3518           }
3519         }
3520 
3521         // check to see if the existence of this region matches the region in META
3522         for (HRegionInfo r:regions) {
3523           HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
3524           hbi.addServer(r, rsinfo);
3525         }
3526       } catch (IOException e) {          // unable to connect to the region server.
3527         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
3528           " Unable to fetch region information. " + e);
3529         throw e;
3530       }
3531       return null;
3532     }
3533 
3534     private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
3535       List<HRegionInfo> ret = Lists.newArrayList();
3536       for (HRegionInfo hri : regions) {
3537         if (hri.isMetaTable() || (!hbck.checkMetaOnly
3538             && hbck.isTableIncluded(hri.getTable()))) {
3539           ret.add(hri);
3540         }
3541       }
3542       return ret;
3543     }
3544   }
3545 
3546   /**
3547    * Contact hdfs and get all information about specified table directory into
3548    * regioninfo list.
3549    */
3550   static class WorkItemHdfsDir implements Callable<Void> {
3551     private HBaseFsck hbck;
3552     private FileStatus tableDir;
3553     private ErrorReporter errors;
3554     private FileSystem fs;
3555 
3556     WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
3557                     FileStatus status) {
3558       this.hbck = hbck;
3559       this.fs = fs;
3560       this.tableDir = status;
3561       this.errors = errors;
3562     }
3563 
3564     @Override
3565     public synchronized Void call() throws IOException {
3566       try {
3567         // level 2: <HBASE_DIR>/<table>/*
3568         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
3569         for (FileStatus regionDir : regionDirs) {
3570           String encodedName = regionDir.getPath().getName();
3571           // ignore directories that aren't hexadecimal
3572           if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
3573             continue;
3574           }
3575 
3576           LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
3577           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
3578           HdfsEntry he = new HdfsEntry();
3579           synchronized (hbi) {
3580             if (hbi.getHdfsRegionDir() != null) {
3581               errors.print("Directory " + encodedName + " duplicate??" +
3582                            hbi.getHdfsRegionDir());
3583             }
3584 
3585             he.hdfsRegionDir = regionDir.getPath();
3586             he.hdfsRegionDirModTime = regionDir.getModificationTime();
3587             Path regioninfoFile = new Path(he.hdfsRegionDir, HRegionFileSystem.REGION_INFO_FILE);
3588             he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
3589             // we add to orphan list when we attempt to read .regioninfo
3590 
3591             // Set a flag if this region contains only edits
3592             // This is special case if a region is left after split
3593             he.hdfsOnlyEdits = true;
3594             FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
3595             Path ePath = HLogUtil.getRegionDirRecoveredEditsDir(regionDir.getPath());
3596             for (FileStatus subDir : subDirs) {
3597               String sdName = subDir.getPath().getName();
3598               if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
3599                 he.hdfsOnlyEdits = false;
3600                 break;
3601               }
3602             }
3603             hbi.hdfsEntry = he;
3604           }
3605         }
3606       } catch (IOException e) {
3607         // unable to connect to the region server.
3608         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
3609             + tableDir.getPath().getName()
3610             + " Unable to fetch region information. " + e);
3611         throw e;
3612       }
3613       return null;
3614     }
3615   }
3616 
3617   /**
3618    * Contact hdfs and get all information about specified table directory into
3619    * regioninfo list.
3620    */
3621   static class WorkItemHdfsRegionInfo implements Callable<Void> {
3622     private HbckInfo hbi;
3623     private HBaseFsck hbck;
3624     private ErrorReporter errors;
3625 
3626     WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
3627       this.hbi = hbi;
3628       this.hbck = hbck;
3629       this.errors = errors;
3630     }
3631 
3632     @Override
3633     public synchronized Void call() throws IOException {
3634       // only load entries that haven't been loaded yet.
3635       if (hbi.getHdfsHRI() == null) {
3636         try {
3637           hbck.loadHdfsRegioninfo(hbi);
3638         } catch (IOException ioe) {
3639           String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
3640               + hbi.getTableName() + " in hdfs dir "
3641               + hbi.getHdfsRegionDir()
3642               + "!  It may be an invalid format or version file.  Treating as "
3643               + "an orphaned regiondir.";
3644           errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
3645           try {
3646             hbck.debugLsr(hbi.getHdfsRegionDir());
3647           } catch (IOException ioe2) {
3648             LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
3649             throw ioe2;
3650           }
3651           hbck.orphanHdfsDirs.add(hbi);
3652           throw ioe;
3653         }
3654       }
3655       return null;
3656     }
3657   };
3658 
3659   /**
3660    * Display the full report from fsck. This displays all live and dead region
3661    * servers, and all known regions.
3662    */
3663   public static void setDisplayFullReport() {
3664     details = true;
3665   }
3666 
3667   /**
3668    * Set summary mode.
3669    * Print only summary of the tables and status (OK or INCONSISTENT)
3670    */
3671   void setSummary() {
3672     summary = true;
3673   }
3674 
3675   /**
3676    * Set hbase:meta check mode.
3677    * Print only info about hbase:meta table deployment/state
3678    */
3679   void setCheckMetaOnly() {
3680     checkMetaOnly = true;
3681   }
3682 
3683   /**
3684    * Set region boundaries check mode.
3685    */
3686   void setRegionBoundariesCheck() {
3687     checkRegionBoundaries = true;
3688   }
3689 
3690   /**
3691    * Set table locks fix mode.
3692    * Delete table locks held for a long time
3693    */
3694   public void setFixTableLocks(boolean shouldFix) {
3695     fixTableLocks = shouldFix;
3696   }
3697 
3698   /**
3699    * Check if we should rerun fsck again. This checks if we've tried to
3700    * fix something and we should rerun fsck tool again.
3701    * Display the full report from fsck. This displays all live and dead
3702    * region servers, and all known regions.
3703    */
3704   void setShouldRerun() {
3705     rerun = true;
3706   }
3707 
3708   boolean shouldRerun() {
3709     return rerun;
3710   }
3711 
3712   /**
3713    * Fix inconsistencies found by fsck. This should try to fix errors (if any)
3714    * found by fsck utility.
3715    */
3716   public void setFixAssignments(boolean shouldFix) {
3717     fixAssignments = shouldFix;
3718   }
3719 
3720   boolean shouldFixAssignments() {
3721     return fixAssignments;
3722   }
3723 
3724   public void setFixMeta(boolean shouldFix) {
3725     fixMeta = shouldFix;
3726   }
3727 
3728   boolean shouldFixMeta() {
3729     return fixMeta;
3730   }
3731 
3732   public void setFixEmptyMetaCells(boolean shouldFix) {
3733     fixEmptyMetaCells = shouldFix;
3734   }
3735 
3736   boolean shouldFixEmptyMetaCells() {
3737     return fixEmptyMetaCells;
3738   }
3739 
3740   public void setCheckHdfs(boolean checking) {
3741     checkHdfs = checking;
3742   }
3743 
3744   boolean shouldCheckHdfs() {
3745     return checkHdfs;
3746   }
3747 
3748   public void setFixHdfsHoles(boolean shouldFix) {
3749     fixHdfsHoles = shouldFix;
3750   }
3751 
3752   boolean shouldFixHdfsHoles() {
3753     return fixHdfsHoles;
3754   }
3755 
3756   public void setFixTableOrphans(boolean shouldFix) {
3757     fixTableOrphans = shouldFix;
3758   }
3759 
3760   boolean shouldFixTableOrphans() {
3761     return fixTableOrphans;
3762   }
3763 
3764   public void setFixHdfsOverlaps(boolean shouldFix) {
3765     fixHdfsOverlaps = shouldFix;
3766   }
3767 
3768   boolean shouldFixHdfsOverlaps() {
3769     return fixHdfsOverlaps;
3770   }
3771 
3772   public void setFixHdfsOrphans(boolean shouldFix) {
3773     fixHdfsOrphans = shouldFix;
3774   }
3775 
3776   boolean shouldFixHdfsOrphans() {
3777     return fixHdfsOrphans;
3778   }
3779 
3780   public void setFixVersionFile(boolean shouldFix) {
3781     fixVersionFile = shouldFix;
3782   }
3783 
3784   public boolean shouldFixVersionFile() {
3785     return fixVersionFile;
3786   }
3787 
3788   public void setSidelineBigOverlaps(boolean sbo) {
3789     this.sidelineBigOverlaps = sbo;
3790   }
3791 
3792   public boolean shouldSidelineBigOverlaps() {
3793     return sidelineBigOverlaps;
3794   }
3795 
3796   public void setFixSplitParents(boolean shouldFix) {
3797     fixSplitParents = shouldFix;
3798   }
3799 
3800   boolean shouldFixSplitParents() {
3801     return fixSplitParents;
3802   }
3803 
3804   public void setFixReferenceFiles(boolean shouldFix) {
3805     fixReferenceFiles = shouldFix;
3806   }
3807 
3808   boolean shouldFixReferenceFiles() {
3809     return fixReferenceFiles;
3810   }
3811 
3812   public boolean shouldIgnorePreCheckPermission() {
3813     return ignorePreCheckPermission;
3814   }
3815 
3816   public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
3817     this.ignorePreCheckPermission = ignorePreCheckPermission;
3818   }
3819 
3820   /**
3821    * @param mm maximum number of regions to merge into a single region.
3822    */
3823   public void setMaxMerge(int mm) {
3824     this.maxMerge = mm;
3825   }
3826 
3827   public int getMaxMerge() {
3828     return maxMerge;
3829   }
3830 
3831   public void setMaxOverlapsToSideline(int mo) {
3832     this.maxOverlapsToSideline = mo;
3833   }
3834 
3835   public int getMaxOverlapsToSideline() {
3836     return maxOverlapsToSideline;
3837   }
3838 
3839   /**
3840    * Only check/fix tables specified by the list,
3841    * Empty list means all tables are included.
3842    */
3843   boolean isTableIncluded(TableName table) {
3844     return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
3845   }
3846 
3847   public void includeTable(TableName table) {
3848     tablesIncluded.add(table);
3849   }
3850 
3851   Set<TableName> getIncludedTables() {
3852     return new HashSet<TableName>(tablesIncluded);
3853   }
3854 
3855   /**
3856    * We are interested in only those tables that have not changed their state in
3857    * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
3858    * @param seconds - the time in seconds
3859    */
3860   public void setTimeLag(long seconds) {
3861     timelag = seconds * 1000; // convert to milliseconds
3862   }
3863 
3864   /**
3865    *
3866    * @param sidelineDir - HDFS path to sideline data
3867    */
3868   public void setSidelineDir(String sidelineDir) {
3869     this.sidelineDir = new Path(sidelineDir);
3870   }
3871 
3872   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
3873     return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
3874   }
3875 
3876   public HFileCorruptionChecker getHFilecorruptionChecker() {
3877     return hfcc;
3878   }
3879 
3880   public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
3881     this.hfcc = hfcc;
3882   }
3883 
3884   public void setRetCode(int code) {
3885     this.retcode = code;
3886   }
3887 
3888   public int getRetCode() {
3889     return retcode;
3890   }
3891 
3892   protected HBaseFsck printUsageAndExit() {
3893     StringWriter sw = new StringWriter(2048);
3894     PrintWriter out = new PrintWriter(sw);
3895     out.println("Usage: fsck [opts] {only tables}");
3896     out.println(" where [opts] are:");
3897     out.println("   -help Display help options (this)");
3898     out.println("   -details Display full report of all regions.");
3899     out.println("   -timelag <timeInSeconds>  Process only regions that " +
3900                        " have not experienced any metadata updates in the last " +
3901                        " <timeInSeconds> seconds.");
3902     out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
3903         " before checking if the fix worked if run with -fix");
3904     out.println("   -summary Print only summary of the tables and status.");
3905     out.println("   -metaonly Only check the state of the hbase:meta table.");
3906     out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
3907     out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
3908 
3909     out.println("");
3910     out.println("  Metadata Repair options: (expert features, use with caution!)");
3911     out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
3912     out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
3913     out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
3914     out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
3915         + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
3916     out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
3917     out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
3918     out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
3919     out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
3920     out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
3921     out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
3922     out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
3923     out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
3924     out.println("   -fixSplitParents  Try to force offline split parents to be online.");
3925     out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
3926     out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
3927     out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
3928         + " (empty REGIONINFO_QUALIFIER rows)");
3929 
3930     out.println("");
3931     out.println("  Datafile Repair options: (expert features, use with caution!)");
3932     out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
3933     out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
3934 
3935     out.println("");
3936     out.println("  Metadata Repair shortcuts");
3937     out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
3938         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles -fixTableLocks");
3939     out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
3940 
3941     out.println("");
3942     out.println("  Table lock options");
3943     out.println("   -fixTableLocks    Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
3944 
3945     out.flush();
3946     errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
3947 
3948     setRetCode(-2);
3949     return this;
3950   }
3951 
3952   /**
3953    * Main program
3954    *
3955    * @param args
3956    * @throws Exception
3957    */
3958   public static void main(String[] args) throws Exception {
3959     // create a fsck object
3960     Configuration conf = HBaseConfiguration.create();
3961     Path hbasedir = FSUtils.getRootDir(conf);
3962     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
3963     FSUtils.setFsDefault(conf, new Path(defaultFs));
3964 
3965     int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
3966     System.exit(ret);
3967   }
3968 
3969   /**
3970    * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
3971    */
3972   static class HBaseFsckTool extends Configured implements Tool {
3973     HBaseFsckTool(Configuration conf) { super(conf); }
3974     @Override
3975     public int run(String[] args) throws Exception {
3976       HBaseFsck hbck = new HBaseFsck(getConf());
3977       hbck.exec(hbck.executor, args);
3978       return hbck.getRetCode();
3979     }
3980   };
3981 
3982 
3983   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
3984     ServiceException, InterruptedException {
3985     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
3986 
3987     boolean checkCorruptHFiles = false;
3988     boolean sidelineCorruptHFiles = false;
3989 
3990     // Process command-line args.
3991     for (int i = 0; i < args.length; i++) {
3992       String cmd = args[i];
3993       if (cmd.equals("-help") || cmd.equals("-h")) {
3994         return printUsageAndExit();
3995       } else if (cmd.equals("-details")) {
3996         setDisplayFullReport();
3997       } else if (cmd.equals("-timelag")) {
3998         if (i == args.length - 1) {
3999           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4000           return printUsageAndExit();
4001         }
4002         try {
4003           long timelag = Long.parseLong(args[i+1]);
4004           setTimeLag(timelag);
4005         } catch (NumberFormatException e) {
4006           errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4007           return printUsageAndExit();
4008         }
4009         i++;
4010       } else if (cmd.equals("-sleepBeforeRerun")) {
4011         if (i == args.length - 1) {
4012           errors.reportError(ERROR_CODE.WRONG_USAGE,
4013             "HBaseFsck: -sleepBeforeRerun needs a value.");
4014           return printUsageAndExit();
4015         }
4016         try {
4017           sleepBeforeRerun = Long.parseLong(args[i+1]);
4018         } catch (NumberFormatException e) {
4019           errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4020           return printUsageAndExit();
4021         }
4022         i++;
4023       } else if (cmd.equals("-sidelineDir")) {
4024         if (i == args.length - 1) {
4025           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4026           return printUsageAndExit();
4027         }
4028         i++;
4029         setSidelineDir(args[i]);
4030       } else if (cmd.equals("-fix")) {
4031         errors.reportError(ERROR_CODE.WRONG_USAGE,
4032           "This option is deprecated, please use  -fixAssignments instead.");
4033         setFixAssignments(true);
4034       } else if (cmd.equals("-fixAssignments")) {
4035         setFixAssignments(true);
4036       } else if (cmd.equals("-fixMeta")) {
4037         setFixMeta(true);
4038       } else if (cmd.equals("-noHdfsChecking")) {
4039         setCheckHdfs(false);
4040       } else if (cmd.equals("-fixHdfsHoles")) {
4041         setFixHdfsHoles(true);
4042       } else if (cmd.equals("-fixHdfsOrphans")) {
4043         setFixHdfsOrphans(true);
4044       } else if (cmd.equals("-fixTableOrphans")) {
4045         setFixTableOrphans(true);
4046       } else if (cmd.equals("-fixHdfsOverlaps")) {
4047         setFixHdfsOverlaps(true);
4048       } else if (cmd.equals("-fixVersionFile")) {
4049         setFixVersionFile(true);
4050       } else if (cmd.equals("-sidelineBigOverlaps")) {
4051         setSidelineBigOverlaps(true);
4052       } else if (cmd.equals("-fixSplitParents")) {
4053         setFixSplitParents(true);
4054       } else if (cmd.equals("-ignorePreCheckPermission")) {
4055         setIgnorePreCheckPermission(true);
4056       } else if (cmd.equals("-checkCorruptHFiles")) {
4057         checkCorruptHFiles = true;
4058       } else if (cmd.equals("-sidelineCorruptHFiles")) {
4059         sidelineCorruptHFiles = true;
4060       } else if (cmd.equals("-fixReferenceFiles")) {
4061         setFixReferenceFiles(true);
4062       } else if (cmd.equals("-fixEmptyMetaCells")) {
4063         setFixEmptyMetaCells(true);
4064       } else if (cmd.equals("-repair")) {
4065         // this attempts to merge overlapping hdfs regions, needs testing
4066         // under load
4067         setFixHdfsHoles(true);
4068         setFixHdfsOrphans(true);
4069         setFixMeta(true);
4070         setFixAssignments(true);
4071         setFixHdfsOverlaps(true);
4072         setFixVersionFile(true);
4073         setSidelineBigOverlaps(true);
4074         setFixSplitParents(false);
4075         setCheckHdfs(true);
4076         setFixReferenceFiles(true);
4077         setFixTableLocks(true);
4078       } else if (cmd.equals("-repairHoles")) {
4079         // this will make all missing hdfs regions available but may lose data
4080         setFixHdfsHoles(true);
4081         setFixHdfsOrphans(false);
4082         setFixMeta(true);
4083         setFixAssignments(true);
4084         setFixHdfsOverlaps(false);
4085         setSidelineBigOverlaps(false);
4086         setFixSplitParents(false);
4087         setCheckHdfs(true);
4088       } else if (cmd.equals("-maxOverlapsToSideline")) {
4089         if (i == args.length - 1) {
4090           errors.reportError(ERROR_CODE.WRONG_USAGE,
4091             "-maxOverlapsToSideline needs a numeric value argument.");
4092           return printUsageAndExit();
4093         }
4094         try {
4095           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4096           setMaxOverlapsToSideline(maxOverlapsToSideline);
4097         } catch (NumberFormatException e) {
4098           errors.reportError(ERROR_CODE.WRONG_USAGE,
4099             "-maxOverlapsToSideline needs a numeric value argument.");
4100           return printUsageAndExit();
4101         }
4102         i++;
4103       } else if (cmd.equals("-maxMerge")) {
4104         if (i == args.length - 1) {
4105           errors.reportError(ERROR_CODE.WRONG_USAGE,
4106             "-maxMerge needs a numeric value argument.");
4107           return printUsageAndExit();
4108         }
4109         try {
4110           int maxMerge = Integer.parseInt(args[i+1]);
4111           setMaxMerge(maxMerge);
4112         } catch (NumberFormatException e) {
4113           errors.reportError(ERROR_CODE.WRONG_USAGE,
4114             "-maxMerge needs a numeric value argument.");
4115           return printUsageAndExit();
4116         }
4117         i++;
4118       } else if (cmd.equals("-summary")) {
4119         setSummary();
4120       } else if (cmd.equals("-metaonly")) {
4121         setCheckMetaOnly();
4122       } else if (cmd.equals("-boundaries")) {
4123         setRegionBoundariesCheck();
4124       } else if (cmd.equals("-fixTableLocks")) {
4125         setFixTableLocks(true);
4126       } else if (cmd.startsWith("-")) {
4127         errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4128         return printUsageAndExit();
4129       } else {
4130         includeTable(TableName.valueOf(cmd));
4131         errors.print("Allow checking/fixes for table: " + cmd);
4132       }
4133     }
4134 
4135     // pre-check current user has FS write permission or not
4136     try {
4137       preCheckPermission();
4138     } catch (AccessDeniedException ace) {
4139       Runtime.getRuntime().exit(-1);
4140     } catch (IOException ioe) {
4141       Runtime.getRuntime().exit(-1);
4142     }
4143 
4144     // do the real work of hbck
4145     connect();
4146 
4147     try {
4148       // if corrupt file mode is on, first fix them since they may be opened later
4149       if (checkCorruptHFiles || sidelineCorruptHFiles) {
4150         LOG.info("Checking all hfiles for corruption");
4151         HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4152         setHFileCorruptionChecker(hfcc); // so we can get result
4153         Collection<TableName> tables = getIncludedTables();
4154         Collection<Path> tableDirs = new ArrayList<Path>();
4155         Path rootdir = FSUtils.getRootDir(getConf());
4156         if (tables.size() > 0) {
4157           for (TableName t : tables) {
4158             tableDirs.add(FSUtils.getTableDir(rootdir, t));
4159           }
4160         } else {
4161           tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4162         }
4163         hfcc.checkTables(tableDirs);
4164         hfcc.report(errors);
4165       }
4166 
4167       // check and fix table integrity, region consistency.
4168       int code = onlineHbck();
4169       setRetCode(code);
4170       // If we have changed the HBase state it is better to run hbck again
4171       // to see if we haven't broken something else in the process.
4172       // We run it only once more because otherwise we can easily fall into
4173       // an infinite loop.
4174       if (shouldRerun()) {
4175         try {
4176           LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4177           Thread.sleep(sleepBeforeRerun);
4178         } catch (InterruptedException ie) {
4179           LOG.warn("Interrupted while sleeping");
4180           return this;
4181         }
4182         // Just report
4183         setFixAssignments(false);
4184         setFixMeta(false);
4185         setFixHdfsHoles(false);
4186         setFixHdfsOverlaps(false);
4187         setFixVersionFile(false);
4188         setFixTableOrphans(false);
4189         errors.resetErrors();
4190         code = onlineHbck();
4191         setRetCode(code);
4192       }
4193     } finally {
4194       IOUtils.cleanup(null, connection, meta, admin);
4195     }
4196     return this;
4197   }
4198 
4199   /**
4200    * ls -r for debugging purposes
4201    */
4202   void debugLsr(Path p) throws IOException {
4203     debugLsr(getConf(), p, errors);
4204   }
4205 
4206   /**
4207    * ls -r for debugging purposes
4208    */
4209   public static void debugLsr(Configuration conf,
4210       Path p) throws IOException {
4211     debugLsr(conf, p, new PrintingErrorReporter());
4212   }
4213 
4214   /**
4215    * ls -r for debugging purposes
4216    */
4217   public static void debugLsr(Configuration conf,
4218       Path p, ErrorReporter errors) throws IOException {
4219     if (!LOG.isDebugEnabled() || p == null) {
4220       return;
4221     }
4222     FileSystem fs = p.getFileSystem(conf);
4223 
4224     if (!fs.exists(p)) {
4225       // nothing
4226       return;
4227     }
4228     errors.print(p.toString());
4229 
4230     if (fs.isFile(p)) {
4231       return;
4232     }
4233 
4234     if (fs.getFileStatus(p).isDirectory()) {
4235       FileStatus[] fss= fs.listStatus(p);
4236       for (FileStatus status : fss) {
4237         debugLsr(conf, status.getPath(), errors);
4238       }
4239     }
4240   }
4241 }