View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.FileNotFoundException;
21  import java.io.IOException;
22  import java.io.PrintWriter;
23  import java.io.StringWriter;
24  import java.net.URI;
25  import java.util.ArrayList;
26  import java.util.Arrays;
27  import java.util.Collection;
28  import java.util.Collections;
29  import java.util.Comparator;
30  import java.util.HashMap;
31  import java.util.HashSet;
32  import java.util.Iterator;
33  import java.util.List;
34  import java.util.Map;
35  import java.util.Map.Entry;
36  import java.util.Set;
37  import java.util.SortedMap;
38  import java.util.SortedSet;
39  import java.util.TreeMap;
40  import java.util.TreeSet;
41  import java.util.concurrent.Callable;
42  import java.util.concurrent.ConcurrentSkipListMap;
43  import java.util.concurrent.ExecutionException;
44  import java.util.concurrent.ExecutorService;
45  import java.util.concurrent.Future;
46  import java.util.concurrent.ScheduledThreadPoolExecutor;
47  import java.util.concurrent.atomic.AtomicInteger;
48  
49  import org.apache.commons.logging.Log;
50  import org.apache.commons.logging.LogFactory;
51  import org.apache.hadoop.classification.InterfaceAudience;
52  import org.apache.hadoop.classification.InterfaceStability;
53  import org.apache.hadoop.conf.Configuration;
54  import org.apache.hadoop.conf.Configured;
55  import org.apache.hadoop.fs.FileStatus;
56  import org.apache.hadoop.fs.FileSystem;
57  import org.apache.hadoop.fs.Path;
58  import org.apache.hadoop.fs.permission.FsAction;
59  import org.apache.hadoop.hbase.Abortable;
60  import org.apache.hadoop.hbase.Cell;
61  import org.apache.hadoop.hbase.ClusterStatus;
62  import org.apache.hadoop.hbase.HBaseConfiguration;
63  import org.apache.hadoop.hbase.HColumnDescriptor;
64  import org.apache.hadoop.hbase.HConstants;
65  import org.apache.hadoop.hbase.HRegionInfo;
66  import org.apache.hadoop.hbase.HRegionLocation;
67  import org.apache.hadoop.hbase.HTableDescriptor;
68  import org.apache.hadoop.hbase.KeyValue;
69  import org.apache.hadoop.hbase.MasterNotRunningException;
70  import org.apache.hadoop.hbase.MetaTableAccessor;
71  import org.apache.hadoop.hbase.RegionLocations;
72  import org.apache.hadoop.hbase.ServerName;
73  import org.apache.hadoop.hbase.TableDescriptor;
74  import org.apache.hadoop.hbase.TableName;
75  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
76  import org.apache.hadoop.hbase.MetaTableAccessor;
77  import org.apache.hadoop.hbase.client.Admin;
78  import org.apache.hadoop.hbase.client.Delete;
79  import org.apache.hadoop.hbase.client.Get;
80  import org.apache.hadoop.hbase.client.HBaseAdmin;
81  import org.apache.hadoop.hbase.client.HConnectable;
82  import org.apache.hadoop.hbase.client.HConnection;
83  import org.apache.hadoop.hbase.client.HConnectionManager;
84  import org.apache.hadoop.hbase.client.HTable;
85  import org.apache.hadoop.hbase.client.MetaScanner;
86  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
87  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
88  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
89  import org.apache.hadoop.hbase.client.Put;
90  import org.apache.hadoop.hbase.client.Result;
91  import org.apache.hadoop.hbase.client.RowMutations;
92  import org.apache.hadoop.hbase.client.Table;
93  import org.apache.hadoop.hbase.client.TableState;
94  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
95  import org.apache.hadoop.hbase.io.hfile.HFile;
96  import org.apache.hadoop.hbase.master.MasterFileSystem;
97  import org.apache.hadoop.hbase.master.RegionState;
98  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
99  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
100 import org.apache.hadoop.hbase.regionserver.HRegion;
101 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
102 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
103 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
104 import org.apache.hadoop.hbase.security.UserProvider;
105 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
106 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
107 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
108 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
109 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
110 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
111 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
112 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
113 import org.apache.hadoop.hbase.security.AccessDeniedException;
114 import org.apache.hadoop.io.IOUtils;
115 import org.apache.hadoop.security.UserGroupInformation;
116 import org.apache.hadoop.util.ReflectionUtils;
117 import org.apache.hadoop.util.Tool;
118 import org.apache.hadoop.util.ToolRunner;
119 import org.apache.zookeeper.KeeperException;
120 
121 import com.google.common.base.Joiner;
122 import com.google.common.base.Preconditions;
123 import com.google.common.collect.Lists;
124 import com.google.common.collect.Multimap;
125 import com.google.common.collect.TreeMultimap;
126 import com.google.protobuf.ServiceException;
127 
128 /**
129  * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
130  * table integrity problems in a corrupted HBase.
131  * <p>
132  * Region consistency checks verify that hbase:meta, region deployment on region
133  * servers and the state of data in HDFS (.regioninfo files) all are in
134  * accordance.
135  * <p>
136  * Table integrity checks verify that all possible row keys resolve to exactly
137  * one region of a table.  This means there are no individual degenerate
138  * or backwards regions; no holes between regions; and that there are no
139  * overlapping regions.
140  * <p>
141  * The general repair strategy works in two phases:
142  * <ol>
143  * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
144  * <li> Repair Region Consistency with hbase:meta and assignments
145  * </ol>
146  * <p>
147  * For table integrity repairs, the tables' region directories are scanned
148  * for .regioninfo files.  Each table's integrity is then verified.  If there
149  * are any orphan regions (regions with no .regioninfo files) or holes, new
150  * regions are fabricated.  Backwards regions are sidelined as well as empty
151  * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
152  * a new region is created and all data is merged into the new region.
153  * <p>
154  * Table integrity repairs deal solely with HDFS and could potentially be done
155  * offline -- the hbase region servers or master do not need to be running.
156  * This phase can eventually be used to completely reconstruct the hbase:meta table in
157  * an offline fashion.
158  * <p>
159  * Region consistency requires three conditions -- 1) valid .regioninfo file
160  * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
161  * and 3) a region is deployed only at the regionserver that was assigned to
162  * with proper state in the master.
163  * <p>
164  * Region consistency repairs require hbase to be online so that hbck can
165  * contact the HBase master and region servers.  The hbck#connect() method must
166  * first be called successfully.  Much of the region consistency information
167  * is transient and less risky to repair.
168  * <p>
169  * If hbck is run from the command line, there are a handful of arguments that
170  * can be used to limit the kinds of repairs hbck will do.  See the code in
171  * {@link #printUsageAndExit()} for more details.
172  */
173 @InterfaceAudience.Public
174 @InterfaceStability.Evolving
175 public class HBaseFsck extends Configured {
176   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
177   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
178   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
179   private static boolean rsSupportsOffline = true;
180   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
181   private static final int DEFAULT_MAX_MERGE = 5;
182   private static final String TO_BE_LOADED = "to_be_loaded";
183 
184   /**********************
185    * Internal resources
186    **********************/
187   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
188   private ClusterStatus status;
189   private HConnection connection;
190   private HBaseAdmin admin;
191   private Table meta;
192   // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
193   protected ExecutorService executor;
194   private long startMillis = System.currentTimeMillis();
195   private HFileCorruptionChecker hfcc;
196   private int retcode = 0;
197 
198   /***********
199    * Options
200    ***********/
201   private static boolean details = false; // do we display the full report
202   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
203   private boolean fixAssignments = false; // fix assignment errors?
204   private boolean fixMeta = false; // fix meta errors?
205   private boolean checkHdfs = true; // load and check fs consistency?
206   private boolean fixHdfsHoles = false; // fix fs holes?
207   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
208   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
209   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
210   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
211   private boolean fixSplitParents = false; // fix lingering split parents
212   private boolean fixReferenceFiles = false; // fix lingering reference store file
213   private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
214   private boolean fixTableLocks = false; // fix table locks which are expired
215 
216   // limit checking/fixes to listed tables, if empty attempt to check/fix all
217   // hbase:meta are always checked
218   private Set<TableName> tablesIncluded = new HashSet<TableName>();
219   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
220   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
221   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
222   private Path sidelineDir = null;
223 
224   private boolean rerun = false; // if we tried to fix something, rerun hbck
225   private static boolean summary = false; // if we want to print less output
226   private boolean checkMetaOnly = false;
227   private boolean checkRegionBoundaries = false;
228   private boolean ignorePreCheckPermission = false; // if pre-check permission
229 
230   /*********
231    * State
232    *********/
233   final private ErrorReporter errors;
234   int fixes = 0;
235 
236   /**
237    * This map contains the state of all hbck items.  It maps from encoded region
238    * name to HbckInfo structure.  The information contained in HbckInfo is used
239    * to detect and correct consistency (hdfs/meta/deployment) problems.
240    */
241   private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
242   private TreeSet<TableName> disabledTables =
243     new TreeSet<TableName>();
244   // Empty regioninfo qualifiers in hbase:meta
245   private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
246 
247   /**
248    * This map from Tablename -> TableInfo contains the structures necessary to
249    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
250    * to prevent dupes.
251    *
252    * If tablesIncluded is empty, this map contains all tables.
253    * Otherwise, it contains only meta tables and tables in tablesIncluded,
254    * unless checkMetaOnly is specified, in which case, it contains only
255    * the meta table
256    */
257   private SortedMap<TableName, TableInfo> tablesInfo =
258       new ConcurrentSkipListMap<TableName, TableInfo>();
259 
260   /**
261    * When initially looking at HDFS, we attempt to find any orphaned data.
262    */
263   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
264 
265   private Map<TableName, Set<String>> orphanTableDirs =
266       new HashMap<TableName, Set<String>>();
267 
268   /**
269    * Constructor
270    *
271    * @param conf Configuration object
272    * @throws MasterNotRunningException if the master is not running
273    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
274    */
275   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
276       ZooKeeperConnectionException, IOException, ClassNotFoundException {
277     super(conf);
278     // make a copy, just to be sure we're not overriding someone else's config
279     setConf(HBaseConfiguration.create(getConf()));
280     // disable blockcache for tool invocation, see HBASE-10500
281     getConf().setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0);
282     errors = getErrorReporter(conf);
283 
284     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
285     executor = new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
286   }
287 
288   /**
289    * Constructor
290    *
291    * @param conf
292    *          Configuration object
293    * @throws MasterNotRunningException
294    *           if the master is not running
295    * @throws ZooKeeperConnectionException
296    *           if unable to connect to ZooKeeper
297    */
298   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
299       ZooKeeperConnectionException, IOException, ClassNotFoundException {
300     super(conf);
301     errors = getErrorReporter(getConf());
302     this.executor = exec;
303   }
304 
305   /**
306    * To repair region consistency, one must call connect() in order to repair
307    * online state.
308    */
309   public void connect() throws IOException {
310     connection = HConnectionManager.createConnection(getConf());
311     admin = new HBaseAdmin(connection);
312     meta = new HTable(TableName.META_TABLE_NAME, connection);
313     status = admin.getClusterStatus();
314   }
315 
316   /**
317    * Get deployed regions according to the region servers.
318    */
319   private void loadDeployedRegions() throws IOException, InterruptedException {
320     // From the master, get a list of all known live region servers
321     Collection<ServerName> regionServers = status.getServers();
322     errors.print("Number of live region servers: " + regionServers.size());
323     if (details) {
324       for (ServerName rsinfo: regionServers) {
325         errors.print("  " + rsinfo.getServerName());
326       }
327     }
328 
329     // From the master, get a list of all dead region servers
330     Collection<ServerName> deadRegionServers = status.getDeadServerNames();
331     errors.print("Number of dead region servers: " + deadRegionServers.size());
332     if (details) {
333       for (ServerName name: deadRegionServers) {
334         errors.print("  " + name);
335       }
336     }
337 
338     // Print the current master name and state
339     errors.print("Master: " + status.getMaster());
340 
341     // Print the list of all backup masters
342     Collection<ServerName> backupMasters = status.getBackupMasters();
343     errors.print("Number of backup masters: " + backupMasters.size());
344     if (details) {
345       for (ServerName name: backupMasters) {
346         errors.print("  " + name);
347       }
348     }
349 
350     errors.print("Average load: " + status.getAverageLoad());
351     errors.print("Number of requests: " + status.getRequestsCount());
352     errors.print("Number of regions: " + status.getRegionsCount());
353 
354     Map<String, RegionState> rits = status.getRegionsInTransition();
355     errors.print("Number of regions in transition: " + rits.size());
356     if (details) {
357       for (RegionState state: rits.values()) {
358         errors.print("  " + state.toDescriptiveString());
359       }
360     }
361 
362     // Determine what's deployed
363     processRegionServers(regionServers);
364   }
365 
366   /**
367    * Clear the current state of hbck.
368    */
369   private void clearState() {
370     // Make sure regionInfo is empty before starting
371     fixes = 0;
372     regionInfoMap.clear();
373     emptyRegionInfoQualifiers.clear();
374     disabledTables.clear();
375     errors.clear();
376     tablesInfo.clear();
377     orphanHdfsDirs.clear();
378   }
379 
380   /**
381    * This repair method analyzes hbase data in hdfs and repairs it to satisfy
382    * the table integrity rules.  HBase doesn't need to be online for this
383    * operation to work.
384    */
385   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
386     // Initial pass to fix orphans.
387     if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
388         || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
389       LOG.info("Loading regioninfos HDFS");
390       // if nothing is happening this should always complete in two iterations.
391       int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
392       int curIter = 0;
393       do {
394         clearState(); // clears hbck state and reset fixes to 0 and.
395         // repair what's on HDFS
396         restoreHdfsIntegrity();
397         curIter++;// limit the number of iterations.
398       } while (fixes > 0 && curIter <= maxIterations);
399 
400       // Repairs should be done in the first iteration and verification in the second.
401       // If there are more than 2 passes, something funny has happened.
402       if (curIter > 2) {
403         if (curIter == maxIterations) {
404           LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
405               + "Tables integrity may not be fully repaired!");
406         } else {
407           LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
408         }
409       }
410     }
411   }
412 
413   /**
414    * This repair method requires the cluster to be online since it contacts
415    * region servers and the masters.  It makes each region's state in HDFS, in
416    * hbase:meta, and deployments consistent.
417    *
418    * @return If > 0 , number of errors detected, if < 0 there was an unrecoverable
419    * error.  If 0, we have a clean hbase.
420    */
421   public int onlineConsistencyRepair() throws IOException, KeeperException,
422     InterruptedException {
423     clearState();
424 
425     // get regions according to what is online on each RegionServer
426     loadDeployedRegions();
427     // check whether hbase:meta is deployed and online
428     recordMetaRegion();
429     // Check if hbase:meta is found only once and in the right place
430     if (!checkMetaRegion()) {
431       String errorMsg = "hbase:meta table is not consistent. ";
432       if (shouldFixAssignments()) {
433         errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
434       } else {
435         errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
436       }
437       errors.reportError(errorMsg + " Exiting...");
438       return -2;
439     }
440     // Not going with further consistency check for tables when hbase:meta itself is not consistent.
441     LOG.info("Loading regionsinfo from the hbase:meta table");
442     boolean success = loadMetaEntries();
443     if (!success) return -1;
444 
445     // Empty cells in hbase:meta?
446     reportEmptyMetaCells();
447 
448     // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
449     if (shouldFixEmptyMetaCells()) {
450       fixEmptyMetaCells();
451     }
452 
453     // get a list of all tables that have not changed recently.
454     if (!checkMetaOnly) {
455       reportTablesInFlux();
456     }
457 
458     // load regiondirs and regioninfos from HDFS
459     if (shouldCheckHdfs()) {
460       loadHdfsRegionDirs();
461       loadHdfsRegionInfos();
462     }
463 
464     // Get disabled tables from ZooKeeper
465     loadDisabledTables();
466 
467     // fix the orphan tables
468     fixOrphanTables();
469 
470     // Check and fix consistency
471     checkAndFixConsistency();
472 
473     // Check integrity (does not fix)
474     checkIntegrity();
475     return errors.getErrorList().size();
476   }
477 
478   /**
479    * Contacts the master and prints out cluster-wide information
480    * @return 0 on success, non-zero on failure
481    */
482   public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
483     // print hbase server version
484     errors.print("Version: " + status.getHBaseVersion());
485     offlineHdfsIntegrityRepair();
486 
487     // turn the balancer off
488     boolean oldBalancer = admin.setBalancerRunning(false, true);
489     try {
490       onlineConsistencyRepair();
491     }
492     finally {
493       admin.setBalancerRunning(oldBalancer, false);
494     }
495 
496     if (checkRegionBoundaries) {
497       checkRegionBoundaries();
498     }
499 
500     offlineReferenceFileRepair();
501 
502     checkAndFixTableLocks();
503 
504     // Print table summary
505     printTableSummary(tablesInfo);
506     return errors.summarize();
507   }
508 
509   public static byte[] keyOnly (byte[] b) {
510     if (b == null)
511       return b;
512     int rowlength = Bytes.toShort(b, 0);
513     byte[] result = new byte[rowlength];
514     System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
515     return result;
516   }
517 
518   private static class RegionBoundariesInformation {
519     public byte [] regionName;
520     public byte [] metaFirstKey;
521     public byte [] metaLastKey;
522     public byte [] storesFirstKey;
523     public byte [] storesLastKey;
524     @Override
525     public String toString () {
526       return "regionName=" + Bytes.toStringBinary(regionName) +
527              "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
528              "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
529              "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
530              "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
531     }
532   }
533 
534   public void checkRegionBoundaries() {
535     try {
536       ByteArrayComparator comparator = new ByteArrayComparator();
537       List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), false);
538       final RegionBoundariesInformation currentRegionBoundariesInformation =
539           new RegionBoundariesInformation();
540       Path hbaseRoot = FSUtils.getRootDir(getConf());
541       for (HRegionInfo regionInfo : regions) {
542         Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
543         currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
544         // For each region, get the start and stop key from the META and compare them to the
545         // same information from the Stores.
546         Path path = new Path(tableDir, regionInfo.getEncodedName());
547         FileSystem fs = path.getFileSystem(getConf());
548         FileStatus[] files = fs.listStatus(path);
549         // For all the column families in this region...
550         byte[] storeFirstKey = null;
551         byte[] storeLastKey = null;
552         for (FileStatus file : files) {
553           String fileName = file.getPath().toString();
554           fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
555           if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
556             FileStatus[] storeFiles = fs.listStatus(file.getPath());
557             // For all the stores in this column family.
558             for (FileStatus storeFile : storeFiles) {
559               HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
560                   getConf()), getConf());
561               if ((reader.getFirstKey() != null)
562                   && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
563                       reader.getFirstKey()) > 0))) {
564                 storeFirstKey = reader.getFirstKey();
565               }
566               if ((reader.getLastKey() != null)
567                   && ((storeLastKey == null) || (comparator.compare(storeLastKey,
568                       reader.getLastKey())) < 0)) {
569                 storeLastKey = reader.getLastKey();
570               }
571               reader.close();
572             }
573           }
574         }
575         currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
576         currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
577         currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
578         currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
579         if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
580           currentRegionBoundariesInformation.metaFirstKey = null;
581         if (currentRegionBoundariesInformation.metaLastKey.length == 0)
582           currentRegionBoundariesInformation.metaLastKey = null;
583 
584         // For a region to be correct, we need the META start key to be smaller or equal to the
585         // smallest start key from all the stores, and the start key from the next META entry to
586         // be bigger than the last key from all the current stores. First region start key is null;
587         // Last region end key is null; some regions can be empty and not have any store.
588 
589         boolean valid = true;
590         // Checking start key.
591         if ((currentRegionBoundariesInformation.storesFirstKey != null)
592             && (currentRegionBoundariesInformation.metaFirstKey != null)) {
593           valid = valid
594               && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
595                 currentRegionBoundariesInformation.metaFirstKey) >= 0;
596         }
597         // Checking stop key.
598         if ((currentRegionBoundariesInformation.storesLastKey != null)
599             && (currentRegionBoundariesInformation.metaLastKey != null)) {
600           valid = valid
601               && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
602                 currentRegionBoundariesInformation.metaLastKey) < 0;
603         }
604         if (!valid) {
605           errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
606             tablesInfo.get(regionInfo.getTable()));
607           LOG.warn("Region's boundaries not alligned between stores and META for:");
608           LOG.warn(currentRegionBoundariesInformation);
609         }
610       }
611     } catch (IOException e) {
612       LOG.error(e);
613     }
614   }
615 
616   /**
617    * Iterates through the list of all orphan/invalid regiondirs.
618    */
619   private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
620     for (HbckInfo hi : orphanHdfsDirs) {
621       LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
622       adoptHdfsOrphan(hi);
623     }
624   }
625 
626   /**
627    * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
628    * these orphans by creating a new region, and moving the column families,
629    * recovered edits, HLogs, into the new region dir.  We determine the region
630    * startkey and endkeys by looking at all of the hfiles inside the column
631    * families to identify the min and max keys. The resulting region will
632    * likely violate table integrity but will be dealt with by merging
633    * overlapping regions.
634    */
635   @SuppressWarnings("deprecation")
636   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
637     Path p = hi.getHdfsRegionDir();
638     FileSystem fs = p.getFileSystem(getConf());
639     FileStatus[] dirs = fs.listStatus(p);
640     if (dirs == null) {
641       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
642           p + ". This dir could probably be deleted.");
643       return ;
644     }
645 
646     TableName tableName = hi.getTableName();
647     TableInfo tableInfo = tablesInfo.get(tableName);
648     Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
649     HTableDescriptor template = tableInfo.getHTD();
650 
651     // find min and max key values
652     Pair<byte[],byte[]> orphanRegionRange = null;
653     for (FileStatus cf : dirs) {
654       String cfName= cf.getPath().getName();
655       // TODO Figure out what the special dirs are
656       if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
657 
658       FileStatus[] hfiles = fs.listStatus(cf.getPath());
659       for (FileStatus hfile : hfiles) {
660         byte[] start, end;
661         HFile.Reader hf = null;
662         try {
663           CacheConfig cacheConf = new CacheConfig(getConf());
664           hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
665           hf.loadFileInfo();
666           KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
667           start = startKv.getRow();
668           KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
669           end = endKv.getRow();
670         } catch (IOException ioe) {
671           LOG.warn("Problem reading orphan file " + hfile + ", skipping");
672           continue;
673         } catch (NullPointerException ioe) {
674           LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
675           continue;
676         } finally {
677           if (hf != null) {
678             hf.close();
679           }
680         }
681 
682         // expand the range to include the range of all hfiles
683         if (orphanRegionRange == null) {
684           // first range
685           orphanRegionRange = new Pair<byte[], byte[]>(start, end);
686         } else {
687           // TODO add test
688 
689           // expand range only if the hfile is wider.
690           if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
691             orphanRegionRange.setFirst(start);
692           }
693           if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
694             orphanRegionRange.setSecond(end);
695           }
696         }
697       }
698     }
699     if (orphanRegionRange == null) {
700       LOG.warn("No data in dir " + p + ", sidelining data");
701       fixes++;
702       sidelineRegionDir(fs, hi);
703       return;
704     }
705     LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
706         Bytes.toString(orphanRegionRange.getSecond()) + ")");
707 
708     // create new region on hdfs.  move data into place.
709     HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond());
710     LOG.info("Creating new region : " + hri);
711     HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
712     Path target = region.getRegionFileSystem().getRegionDir();
713 
714     // rename all the data to new region
715     mergeRegionDirs(target, hi);
716     fixes++;
717   }
718 
719   /**
720    * This method determines if there are table integrity errors in HDFS.  If
721    * there are errors and the appropriate "fix" options are enabled, the method
722    * will first correct orphan regions making them into legit regiondirs, and
723    * then reload to merge potentially overlapping regions.
724    *
725    * @return number of table integrity errors found
726    */
727   private int restoreHdfsIntegrity() throws IOException, InterruptedException {
728     // Determine what's on HDFS
729     LOG.info("Loading HBase regioninfo from HDFS...");
730     loadHdfsRegionDirs(); // populating regioninfo table.
731 
732     int errs = errors.getErrorList().size();
733     // First time just get suggestions.
734     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
735     checkHdfsIntegrity(false, false);
736 
737     if (errors.getErrorList().size() == errs) {
738       LOG.info("No integrity errors.  We are done with this phase. Glorious.");
739       return 0;
740     }
741 
742     if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
743       adoptHdfsOrphans(orphanHdfsDirs);
744       // TODO optimize by incrementally adding instead of reloading.
745     }
746 
747     // Make sure there are no holes now.
748     if (shouldFixHdfsHoles()) {
749       clearState(); // this also resets # fixes.
750       loadHdfsRegionDirs();
751       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
752       tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
753     }
754 
755     // Now we fix overlaps
756     if (shouldFixHdfsOverlaps()) {
757       // second pass we fix overlaps.
758       clearState(); // this also resets # fixes.
759       loadHdfsRegionDirs();
760       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
761       tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
762     }
763 
764     return errors.getErrorList().size();
765   }
766 
767   /**
768    * Scan all the store file names to find any lingering reference files,
769    * which refer to some none-exiting files. If "fix" option is enabled,
770    * any lingering reference file will be sidelined if found.
771    * <p>
772    * Lingering reference file prevents a region from opening. It has to
773    * be fixed before a cluster can start properly.
774    */
775   private void offlineReferenceFileRepair() throws IOException {
776     Configuration conf = getConf();
777     Path hbaseRoot = FSUtils.getRootDir(conf);
778     FileSystem fs = hbaseRoot.getFileSystem(conf);
779     Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot);
780     for (Path path: allFiles.values()) {
781       boolean isReference = false;
782       try {
783         isReference = StoreFileInfo.isReference(path);
784       } catch (Throwable t) {
785         // Ignore. Some files may not be store files at all.
786         // For example, files under .oldlogs folder in hbase:meta
787         // Warning message is already logged by
788         // StoreFile#isReference.
789       }
790       if (!isReference) continue;
791 
792       Path referredToFile = StoreFileInfo.getReferredToFile(path);
793       if (fs.exists(referredToFile)) continue;  // good, expected
794 
795       // Found a lingering reference file
796       errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
797         "Found lingering reference file " + path);
798       if (!shouldFixReferenceFiles()) continue;
799 
800       // Now, trying to fix it since requested
801       boolean success = false;
802       String pathStr = path.toString();
803 
804       // A reference file path should be like
805       // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
806       // Up 5 directories to get the root folder.
807       // So the file will be sidelined to a similar folder structure.
808       int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
809       for (int i = 0; index > 0 && i < 5; i++) {
810         index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
811       }
812       if (index > 0) {
813         Path rootDir = getSidelineDir();
814         Path dst = new Path(rootDir, pathStr.substring(index + 1));
815         fs.mkdirs(dst.getParent());
816         LOG.info("Trying to sildeline reference file "
817           + path + " to " + dst);
818         setShouldRerun();
819 
820         success = fs.rename(path, dst);
821       }
822       if (!success) {
823         LOG.error("Failed to sideline reference file " + path);
824       }
825     }
826   }
827 
828   /**
829    * TODO -- need to add tests for this.
830    */
831   private void reportEmptyMetaCells() {
832     errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
833       emptyRegionInfoQualifiers.size());
834     if (details) {
835       for (Result r: emptyRegionInfoQualifiers) {
836         errors.print("  " + r);
837       }
838     }
839   }
840 
841   /**
842    * TODO -- need to add tests for this.
843    */
844   private void reportTablesInFlux() {
845     AtomicInteger numSkipped = new AtomicInteger(0);
846     HTableDescriptor[] allTables = getTables(numSkipped);
847     errors.print("Number of Tables: " + allTables.length);
848     if (details) {
849       if (numSkipped.get() > 0) {
850         errors.detail("Number of Tables in flux: " + numSkipped.get());
851       }
852       for (HTableDescriptor td : allTables) {
853         errors.detail("  Table: " + td.getTableName() + "\t" +
854                            (td.isReadOnly() ? "ro" : "rw") + "\t" +
855                             (td.isMetaRegion() ? "META" : "    ") + "\t" +
856                            " families: " + td.getFamilies().size());
857       }
858     }
859   }
860 
861   public ErrorReporter getErrors() {
862     return errors;
863   }
864 
865   /**
866    * Read the .regioninfo file from the file system.  If there is no
867    * .regioninfo, add it to the orphan hdfs region list.
868    */
869   private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
870     Path regionDir = hbi.getHdfsRegionDir();
871     if (regionDir == null) {
872       LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
873       return;
874     }
875 
876     if (hbi.hdfsEntry.hri != null) {
877       // already loaded data
878       return;
879     }
880 
881     FileSystem fs = FileSystem.get(getConf());
882     HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
883     LOG.debug("HRegionInfo read: " + hri.toString());
884     hbi.hdfsEntry.hri = hri;
885   }
886 
887   /**
888    * Exception thrown when a integrity repair operation fails in an
889    * unresolvable way.
890    */
891   public static class RegionRepairException extends IOException {
892     private static final long serialVersionUID = 1L;
893     final IOException ioe;
894     public RegionRepairException(String s, IOException ioe) {
895       super(s);
896       this.ioe = ioe;
897     }
898   }
899 
900   /**
901    * Populate hbi's from regionInfos loaded from file system.
902    */
903   private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
904       throws IOException, InterruptedException {
905     tablesInfo.clear(); // regenerating the data
906     // generate region split structure
907     Collection<HbckInfo> hbckInfos = regionInfoMap.values();
908 
909     // Parallelized read of .regioninfo files.
910     List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
911     List<Future<Void>> hbiFutures;
912 
913     for (HbckInfo hbi : hbckInfos) {
914       WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
915       hbis.add(work);
916     }
917 
918     // Submit and wait for completion
919     hbiFutures = executor.invokeAll(hbis);
920 
921     for(int i=0; i<hbiFutures.size(); i++) {
922       WorkItemHdfsRegionInfo work = hbis.get(i);
923       Future<Void> f = hbiFutures.get(i);
924       try {
925         f.get();
926       } catch(ExecutionException e) {
927         LOG.warn("Failed to read .regioninfo file for region " +
928               work.hbi.getRegionNameAsString(), e.getCause());
929       }
930     }
931 
932     Path hbaseRoot = FSUtils.getRootDir(getConf());
933     FileSystem fs = hbaseRoot.getFileSystem(getConf());
934     // serialized table info gathering.
935     for (HbckInfo hbi: hbckInfos) {
936 
937       if (hbi.getHdfsHRI() == null) {
938         // was an orphan
939         continue;
940       }
941 
942 
943       // get table name from hdfs, populate various HBaseFsck tables.
944       TableName tableName = hbi.getTableName();
945       if (tableName == null) {
946         // There was an entry in hbase:meta not in the HDFS?
947         LOG.warn("tableName was null for: " + hbi);
948         continue;
949       }
950 
951       TableInfo modTInfo = tablesInfo.get(tableName);
952       if (modTInfo == null) {
953         // only executed once per table.
954         modTInfo = new TableInfo(tableName);
955         tablesInfo.put(tableName, modTInfo);
956         try {
957           TableDescriptor htd =
958               FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
959           modTInfo.htds.add(htd.getHTableDescriptor());
960         } catch (IOException ioe) {
961           if (!orphanTableDirs.containsKey(tableName)) {
962             LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
963             //should only report once for each table
964             errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
965                 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
966             Set<String> columns = new HashSet<String>();
967             orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
968           }
969         }
970       }
971       if (!hbi.isSkipChecks()) {
972         modTInfo.addRegionInfo(hbi);
973       }
974     }
975 
976     loadTableInfosForTablesWithNoRegion();
977 
978     return tablesInfo;
979   }
980 
981   /**
982    * To get the column family list according to the column family dirs
983    * @param columns
984    * @param hbi
985    * @return a set of column families
986    * @throws IOException
987    */
988   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
989     Path regionDir = hbi.getHdfsRegionDir();
990     FileSystem fs = regionDir.getFileSystem(getConf());
991     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
992     for (FileStatus subdir : subDirs) {
993       String columnfamily = subdir.getPath().getName();
994       columns.add(columnfamily);
995     }
996     return columns;
997   }
998 
999   /**
1000    * To fabricate a .tableinfo file with following contents<br>
1001    * 1. the correct tablename <br>
1002    * 2. the correct colfamily list<br>
1003    * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1004    * @throws IOException
1005    */
1006   private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1007       Set<String> columns) throws IOException {
1008     if (columns ==null || columns.isEmpty()) return false;
1009     HTableDescriptor htd = new HTableDescriptor(tableName);
1010     for (String columnfamimly : columns) {
1011       htd.addFamily(new HColumnDescriptor(columnfamimly));
1012     }
1013     fstd.createTableDescriptor(new TableDescriptor(htd, TableState.State.ENABLED), true);
1014     return true;
1015   }
1016 
1017   /**
1018    * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1019    * @throws IOException
1020    */
1021   public void fixEmptyMetaCells() throws IOException {
1022     if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1023       LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1024       for (Result region : emptyRegionInfoQualifiers) {
1025         deleteMetaRegion(region.getRow());
1026         errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1027       }
1028       emptyRegionInfoQualifiers.clear();
1029     }
1030   }
1031 
1032   /**
1033    * To fix orphan table by creating a .tableinfo file under tableDir <br>
1034    * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1035    * 2. else create a default .tableinfo file with following items<br>
1036    * &nbsp;2.1 the correct tablename <br>
1037    * &nbsp;2.2 the correct colfamily list<br>
1038    * &nbsp;2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1039    * @throws IOException
1040    */
1041   public void fixOrphanTables() throws IOException {
1042     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1043 
1044       List<TableName> tmpList = new ArrayList<TableName>();
1045       tmpList.addAll(orphanTableDirs.keySet());
1046       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1047       Iterator<Entry<TableName, Set<String>>> iter =
1048           orphanTableDirs.entrySet().iterator();
1049       int j = 0;
1050       int numFailedCase = 0;
1051       FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1052       while (iter.hasNext()) {
1053         Entry<TableName, Set<String>> entry =
1054             (Entry<TableName, Set<String>>) iter.next();
1055         TableName tableName = entry.getKey();
1056         LOG.info("Trying to fix orphan table error: " + tableName);
1057         if (j < htds.length) {
1058           if (tableName.equals(htds[j].getTableName())) {
1059             HTableDescriptor htd = htds[j];
1060             LOG.info("fixing orphan table: " + tableName + " from cache");
1061             fstd.createTableDescriptor(new TableDescriptor(htd, TableState.State.ENABLED), true);
1062             j++;
1063             iter.remove();
1064           }
1065         } else {
1066           if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1067             LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1068             LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1069             iter.remove();
1070           } else {
1071             LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1072             numFailedCase++;
1073           }
1074         }
1075         fixes++;
1076       }
1077 
1078       if (orphanTableDirs.isEmpty()) {
1079         // all orphanTableDirs are luckily recovered
1080         // re-run doFsck after recovering the .tableinfo file
1081         setShouldRerun();
1082         LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1083       } else if (numFailedCase > 0) {
1084         LOG.error("Failed to fix " + numFailedCase
1085             + " OrphanTables with default .tableinfo files");
1086       }
1087 
1088     }
1089     //cleanup the list
1090     orphanTableDirs.clear();
1091 
1092   }
1093 
1094   /**
1095    * This borrows code from MasterFileSystem.bootstrap()
1096    *
1097    * @return an open hbase:meta HRegion
1098    */
1099   private HRegion createNewMeta() throws IOException {
1100       Path rootdir = FSUtils.getRootDir(getConf());
1101     Configuration c = getConf();
1102     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1103     MasterFileSystem.setInfoFamilyCachingForMeta(false);
1104     HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c,
1105         HTableDescriptor.META_TABLEDESC);
1106     MasterFileSystem.setInfoFamilyCachingForMeta(true);
1107     return meta;
1108   }
1109 
1110   /**
1111    * Generate set of puts to add to new meta.  This expects the tables to be
1112    * clean with no overlaps or holes.  If there are any problems it returns null.
1113    *
1114    * @return An array list of puts to do in bulk, null if tables have problems
1115    */
1116   private ArrayList<Put> generatePuts(
1117       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1118     ArrayList<Put> puts = new ArrayList<Put>();
1119     boolean hasProblems = false;
1120     for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1121       TableName name = e.getKey();
1122 
1123       // skip "hbase:meta"
1124       if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1125         continue;
1126       }
1127 
1128       TableInfo ti = e.getValue();
1129       for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1130           .entrySet()) {
1131         Collection<HbckInfo> his = spl.getValue();
1132         int sz = his.size();
1133         if (sz != 1) {
1134           // problem
1135           LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1136               + " had " +  sz + " regions instead of exactly 1." );
1137           hasProblems = true;
1138           continue;
1139         }
1140 
1141         // add the row directly to meta.
1142         HbckInfo hi = his.iterator().next();
1143         HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
1144         Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1145         puts.add(p);
1146       }
1147     }
1148     return hasProblems ? null : puts;
1149   }
1150 
1151   /**
1152    * Suggest fixes for each table
1153    */
1154   private void suggestFixes(
1155       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1156     for (TableInfo tInfo : tablesInfo.values()) {
1157       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1158       tInfo.checkRegionChain(handler);
1159     }
1160   }
1161 
1162   /**
1163    * Rebuilds meta from information in hdfs/fs.  Depends on configuration
1164    * settings passed into hbck constructor to point to a particular fs/dir.
1165    *
1166    * @param fix flag that determines if method should attempt to fix holes
1167    * @return true if successful, false if attempt failed.
1168    */
1169   public boolean rebuildMeta(boolean fix) throws IOException,
1170       InterruptedException {
1171 
1172     // TODO check to make sure hbase is offline. (or at least the table
1173     // currently being worked on is off line)
1174 
1175     // Determine what's on HDFS
1176     LOG.info("Loading HBase regioninfo from HDFS...");
1177     loadHdfsRegionDirs(); // populating regioninfo table.
1178 
1179     int errs = errors.getErrorList().size();
1180     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1181     checkHdfsIntegrity(false, false);
1182 
1183     // make sure ok.
1184     if (errors.getErrorList().size() != errs) {
1185       // While in error state, iterate until no more fixes possible
1186       while(true) {
1187         fixes = 0;
1188         suggestFixes(tablesInfo);
1189         errors.clear();
1190         loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1191         checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1192 
1193         int errCount = errors.getErrorList().size();
1194 
1195         if (fixes == 0) {
1196           if (errCount > 0) {
1197             return false; // failed to fix problems.
1198           } else {
1199             break; // no fixes and no problems? drop out and fix stuff!
1200           }
1201         }
1202       }
1203     }
1204 
1205     // we can rebuild, move old meta out of the way and start
1206     LOG.info("HDFS regioninfo's seems good.  Sidelining old hbase:meta");
1207     Path backupDir = sidelineOldMeta();
1208 
1209     LOG.info("Creating new hbase:meta");
1210     HRegion meta = createNewMeta();
1211 
1212     // populate meta
1213     List<Put> puts = generatePuts(tablesInfo);
1214     if (puts == null) {
1215       LOG.fatal("Problem encountered when creating new hbase:meta entries.  " +
1216         "You may need to restore the previously sidelined hbase:meta");
1217       return false;
1218     }
1219     meta.batchMutate(puts.toArray(new Put[0]));
1220     HRegion.closeHRegion(meta);
1221     LOG.info("Success! hbase:meta table rebuilt.");
1222     LOG.info("Old hbase:meta is moved into " + backupDir);
1223     return true;
1224   }
1225 
1226   private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1227       boolean fixOverlaps) throws IOException {
1228     LOG.info("Checking HBase region split map from HDFS data...");
1229     for (TableInfo tInfo : tablesInfo.values()) {
1230       TableIntegrityErrorHandler handler;
1231       if (fixHoles || fixOverlaps) {
1232         handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1233           fixHoles, fixOverlaps);
1234       } else {
1235         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1236       }
1237       if (!tInfo.checkRegionChain(handler)) {
1238         // should dump info as well.
1239         errors.report("Found inconsistency in table " + tInfo.getName());
1240       }
1241     }
1242     return tablesInfo;
1243   }
1244 
1245   private Path getSidelineDir() throws IOException {
1246     if (sidelineDir == null) {
1247       Path hbaseDir = FSUtils.getRootDir(getConf());
1248       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1249       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1250           + startMillis);
1251     }
1252     return sidelineDir;
1253   }
1254 
1255   /**
1256    * Sideline a region dir (instead of deleting it)
1257    */
1258   Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1259     return sidelineRegionDir(fs, null, hi);
1260   }
1261 
1262   /**
1263    * Sideline a region dir (instead of deleting it)
1264    *
1265    * @param parentDir if specified, the region will be sidelined to
1266    * folder like .../parentDir/<table name>/<region name>. The purpose
1267    * is to group together similar regions sidelined, for example, those
1268    * regions should be bulk loaded back later on. If null, it is ignored.
1269    */
1270   Path sidelineRegionDir(FileSystem fs,
1271       String parentDir, HbckInfo hi) throws IOException {
1272     TableName tableName = hi.getTableName();
1273     Path regionDir = hi.getHdfsRegionDir();
1274 
1275     if (!fs.exists(regionDir)) {
1276       LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1277       return null;
1278     }
1279 
1280     Path rootDir = getSidelineDir();
1281     if (parentDir != null) {
1282       rootDir = new Path(rootDir, parentDir);
1283     }
1284     Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1285     Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1286     fs.mkdirs(sidelineRegionDir);
1287     boolean success = false;
1288     FileStatus[] cfs =  fs.listStatus(regionDir);
1289     if (cfs == null) {
1290       LOG.info("Region dir is empty: " + regionDir);
1291     } else {
1292       for (FileStatus cf : cfs) {
1293         Path src = cf.getPath();
1294         Path dst =  new Path(sidelineRegionDir, src.getName());
1295         if (fs.isFile(src)) {
1296           // simple file
1297           success = fs.rename(src, dst);
1298           if (!success) {
1299             String msg = "Unable to rename file " + src +  " to " + dst;
1300             LOG.error(msg);
1301             throw new IOException(msg);
1302           }
1303           continue;
1304         }
1305 
1306         // is a directory.
1307         fs.mkdirs(dst);
1308 
1309         LOG.info("Sidelining files from " + src + " into containing region " + dst);
1310         // FileSystem.rename is inconsistent with directories -- if the
1311         // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1312         // it moves the src into the dst dir resulting in (foo/a/b).  If
1313         // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1314         FileStatus[] hfiles = fs.listStatus(src);
1315         if (hfiles != null && hfiles.length > 0) {
1316           for (FileStatus hfile : hfiles) {
1317             success = fs.rename(hfile.getPath(), dst);
1318             if (!success) {
1319               String msg = "Unable to rename file " + src +  " to " + dst;
1320               LOG.error(msg);
1321               throw new IOException(msg);
1322             }
1323           }
1324         }
1325         LOG.debug("Sideline directory contents:");
1326         debugLsr(sidelineRegionDir);
1327       }
1328     }
1329 
1330     LOG.info("Removing old region dir: " + regionDir);
1331     success = fs.delete(regionDir, true);
1332     if (!success) {
1333       String msg = "Unable to delete dir " + regionDir;
1334       LOG.error(msg);
1335       throw new IOException(msg);
1336     }
1337     return sidelineRegionDir;
1338   }
1339 
1340   /**
1341    * Side line an entire table.
1342    */
1343   void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1344       Path backupHbaseDir) throws IOException {
1345     Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1346     if (fs.exists(tableDir)) {
1347       Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1348       fs.mkdirs(backupTableDir.getParent());
1349       boolean success = fs.rename(tableDir, backupTableDir);
1350       if (!success) {
1351         throw new IOException("Failed to move  " + tableName + " from "
1352             +  tableDir + " to " + backupTableDir);
1353       }
1354     } else {
1355       LOG.info("No previous " + tableName +  " exists.  Continuing.");
1356     }
1357   }
1358 
1359   /**
1360    * @return Path to backup of original directory
1361    */
1362   Path sidelineOldMeta() throws IOException {
1363     // put current hbase:meta aside.
1364     Path hbaseDir = FSUtils.getRootDir(getConf());
1365     FileSystem fs = hbaseDir.getFileSystem(getConf());
1366     Path backupDir = getSidelineDir();
1367     fs.mkdirs(backupDir);
1368 
1369     try {
1370       sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1371     } catch (IOException e) {
1372         LOG.fatal("... failed to sideline meta. Currently in inconsistent state.  To restore "
1373             + "try to rename hbase:meta in " + backupDir.getName() + " to "
1374             + hbaseDir.getName() + ".", e);
1375       throw e; // throw original exception
1376     }
1377     return backupDir;
1378   }
1379 
1380   /**
1381    * Load the list of disabled tables in ZK into local set.
1382    * @throws ZooKeeperConnectionException
1383    * @throws IOException
1384    */
1385   private void loadDisabledTables()
1386   throws IOException {
1387     HConnectionManager.execute(new HConnectable<Void>(getConf()) {
1388       @Override
1389       public Void connect(HConnection connection) throws IOException {
1390         TableName[] tables = connection.listTableNames();
1391         for (TableName table : tables) {
1392           if (connection.getTableState(table)
1393               .inStates(TableState.State.DISABLED, TableState.State.DISABLING)) {
1394             disabledTables.add(table);
1395           }
1396         }
1397         return null;
1398       }
1399     });
1400   }
1401 
1402   /**
1403    * Check if the specified region's table is disabled.
1404    */
1405   private boolean isTableDisabled(HRegionInfo regionInfo) {
1406     return disabledTables.contains(regionInfo.getTable());
1407   }
1408 
1409   /**
1410    * Scan HDFS for all regions, recording their information into
1411    * regionInfoMap
1412    */
1413   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1414     Path rootDir = FSUtils.getRootDir(getConf());
1415     FileSystem fs = rootDir.getFileSystem(getConf());
1416 
1417     // list all tables from HDFS
1418     List<FileStatus> tableDirs = Lists.newArrayList();
1419 
1420     boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1421 
1422     List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1423     for (Path path : paths) {
1424       TableName tableName = FSUtils.getTableName(path);
1425        if ((!checkMetaOnly &&
1426            isTableIncluded(tableName)) ||
1427            tableName.equals(TableName.META_TABLE_NAME)) {
1428          tableDirs.add(fs.getFileStatus(path));
1429        }
1430     }
1431 
1432     // verify that version file exists
1433     if (!foundVersionFile) {
1434       errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1435           "Version file does not exist in root dir " + rootDir);
1436       if (shouldFixVersionFile()) {
1437         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1438             + " file.");
1439         setShouldRerun();
1440         FSUtils.setVersion(fs, rootDir, getConf().getInt(
1441             HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1442             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1443             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1444       }
1445     }
1446 
1447     // level 1:  <HBASE_DIR>/*
1448     List<WorkItemHdfsDir> dirs = new ArrayList<WorkItemHdfsDir>(tableDirs.size());
1449     List<Future<Void>> dirsFutures;
1450 
1451     for (FileStatus tableDir : tableDirs) {
1452       LOG.debug("Loading region dirs from " +tableDir.getPath());
1453       dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir));
1454     }
1455 
1456     // Invoke and wait for Callables to complete
1457     dirsFutures = executor.invokeAll(dirs);
1458 
1459     for(Future<Void> f: dirsFutures) {
1460       try {
1461         f.get();
1462       } catch(ExecutionException e) {
1463         LOG.warn("Could not load region dir " , e.getCause());
1464       }
1465     }
1466   }
1467 
1468   /**
1469    * Record the location of the hbase:meta region as found in ZooKeeper.
1470    */
1471   private boolean recordMetaRegion() throws IOException {
1472     HRegionLocation metaLocation = connection.locateRegion(
1473       TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW);
1474 
1475     // Check if Meta region is valid and existing
1476     if (metaLocation == null || metaLocation.getRegionInfo() == null ||
1477         metaLocation.getHostname() == null) {
1478       errors.reportError(ERROR_CODE.NULL_META_REGION,
1479         "META region or some of its attributes are null.");
1480       return false;
1481     }
1482     ServerName sn;
1483     try {
1484       sn = getMetaRegionServerName();
1485     } catch (KeeperException e) {
1486       throw new IOException(e);
1487     }
1488     MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, System.currentTimeMillis());
1489     HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1490     if (hbckInfo == null) {
1491       regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1492     } else {
1493       hbckInfo.metaEntry = m;
1494     }
1495     return true;
1496   }
1497 
1498   private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1499     return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1500       @Override
1501       public void abort(String why, Throwable e) {
1502         LOG.error(why, e);
1503         System.exit(1);
1504       }
1505 
1506       @Override
1507       public boolean isAborted() {
1508         return false;
1509       }
1510 
1511     });
1512   }
1513 
1514   private ServerName getMetaRegionServerName()
1515   throws IOException, KeeperException {
1516     ZooKeeperWatcher zkw = createZooKeeperWatcher();
1517     ServerName sn = null;
1518     try {
1519       sn = new MetaTableLocator().getMetaRegionLocation(zkw);
1520     } finally {
1521       zkw.close();
1522     }
1523     return sn;
1524   }
1525 
1526   /**
1527    * Contacts each regionserver and fetches metadata about regions.
1528    * @param regionServerList - the list of region servers to connect to
1529    * @throws IOException if a remote or network exception occurs
1530    */
1531   void processRegionServers(Collection<ServerName> regionServerList)
1532     throws IOException, InterruptedException {
1533 
1534     List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1535     List<Future<Void>> workFutures;
1536 
1537     // loop to contact each region server in parallel
1538     for (ServerName rsinfo: regionServerList) {
1539       workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1540     }
1541 
1542     workFutures = executor.invokeAll(workItems);
1543 
1544     for(int i=0; i<workFutures.size(); i++) {
1545       WorkItemRegion item = workItems.get(i);
1546       Future<Void> f = workFutures.get(i);
1547       try {
1548         f.get();
1549       } catch(ExecutionException e) {
1550         LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1551             e.getCause());
1552       }
1553     }
1554   }
1555 
1556   /**
1557    * Check consistency of all regions that have been found in previous phases.
1558    */
1559   private void checkAndFixConsistency()
1560   throws IOException, KeeperException, InterruptedException {
1561 	  // Divide the checks in two phases. One for default/primary replicas and another
1562 	  // for the non-primary ones. Keeps code cleaner this way.
1563     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1564       if (e.getValue().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1565         checkRegionConsistency(e.getKey(), e.getValue());
1566       }
1567     }
1568     boolean prevHdfsCheck = shouldCheckHdfs();
1569     setCheckHdfs(false); //replicas don't have any hdfs data
1570     // Run a pass over the replicas and fix any assignment issues that exist on the currently
1571     // deployed/undeployed replicas.
1572     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1573       if (e.getValue().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1574         checkRegionConsistency(e.getKey(), e.getValue());
1575       }
1576     }
1577     setCheckHdfs(prevHdfsCheck);
1578   }
1579 
1580   private void preCheckPermission() throws IOException, AccessDeniedException {
1581     if (shouldIgnorePreCheckPermission()) {
1582       return;
1583     }
1584 
1585     Path hbaseDir = FSUtils.getRootDir(getConf());
1586     FileSystem fs = hbaseDir.getFileSystem(getConf());
1587     UserProvider userProvider = UserProvider.instantiate(getConf());
1588     UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1589     FileStatus[] files = fs.listStatus(hbaseDir);
1590     for (FileStatus file : files) {
1591       try {
1592         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1593       } catch (AccessDeniedException ace) {
1594         LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);
1595         errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1596           + " does not have write perms to " + file.getPath()
1597           + ". Please rerun hbck as hdfs user " + file.getOwner());
1598         throw ace;
1599       }
1600     }
1601   }
1602 
1603   /**
1604    * Deletes region from meta table
1605    */
1606   private void deleteMetaRegion(HbckInfo hi) throws IOException {
1607     deleteMetaRegion(hi.metaEntry.getRegionName());
1608   }
1609 
1610   /**
1611    * Deletes region from meta table
1612    */
1613   private void deleteMetaRegion(byte[] metaKey) throws IOException {
1614     Delete d = new Delete(metaKey);
1615     meta.delete(d);
1616     meta.flushCommits();
1617     LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1618   }
1619 
1620   /**
1621    * Reset the split parent region info in meta table
1622    */
1623   private void resetSplitParent(HbckInfo hi) throws IOException {
1624     RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
1625     Delete d = new Delete(hi.metaEntry.getRegionName());
1626     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1627     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1628     mutations.add(d);
1629 
1630     HRegionInfo hri = new HRegionInfo(hi.metaEntry);
1631     hri.setOffline(false);
1632     hri.setSplit(false);
1633     Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1634     mutations.add(p);
1635 
1636     meta.mutateRow(mutations);
1637     meta.flushCommits();
1638     LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
1639   }
1640 
1641   /**
1642    * This backwards-compatibility wrapper for permanently offlining a region
1643    * that should not be alive.  If the region server does not support the
1644    * "offline" method, it will use the closest unassign method instead.  This
1645    * will basically work until one attempts to disable or delete the affected
1646    * table.  The problem has to do with in-memory only master state, so
1647    * restarting the HMaster or failing over to another should fix this.
1648    */
1649   private void offline(byte[] regionName) throws IOException {
1650     String regionString = Bytes.toStringBinary(regionName);
1651     if (!rsSupportsOffline) {
1652       LOG.warn("Using unassign region " + regionString
1653           + " instead of using offline method, you should"
1654           + " restart HMaster after these repairs");
1655       admin.unassign(regionName, true);
1656       return;
1657     }
1658 
1659     // first time we assume the rs's supports #offline.
1660     try {
1661       LOG.info("Offlining region " + regionString);
1662       admin.offline(regionName);
1663     } catch (IOException ioe) {
1664       String notFoundMsg = "java.lang.NoSuchMethodException: " +
1665         "org.apache.hadoop.hbase.master.HMaster.offline([B)";
1666       if (ioe.getMessage().contains(notFoundMsg)) {
1667         LOG.warn("Using unassign region " + regionString
1668             + " instead of using offline method, you should"
1669             + " restart HMaster after these repairs");
1670         rsSupportsOffline = false; // in the future just use unassign
1671         admin.unassign(regionName, true);
1672         return;
1673       }
1674       throw ioe;
1675     }
1676   }
1677 
1678   private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
1679     undeployRegionsForHbi(hi);
1680     // undeploy replicas of the region (but only if the method is invoked for the primary)
1681     if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1682       return;
1683     }
1684     int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
1685     for (int i = 1; i < numReplicas; i++) {
1686       if (hi.getPrimaryHRIForDeployedReplica() == null) continue;
1687       HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(
1688           hi.getPrimaryHRIForDeployedReplica(), i);
1689       HbckInfo h = regionInfoMap.get(hri.getEncodedName());
1690       if (h != null) {
1691         undeployRegionsForHbi(h);
1692         //set skip checks; we undeployed it, and we don't want to evaluate this anymore
1693         //in consistency checks
1694         h.setSkipChecks(true);
1695       }
1696     }
1697   }
1698 
1699   private void undeployRegionsForHbi(HbckInfo hi) throws IOException, InterruptedException {
1700     for (OnlineEntry rse : hi.deployedEntries) {
1701       LOG.debug("Undeploy region "  + rse.hri + " from " + rse.hsa);
1702       try {
1703         HBaseFsckRepair.closeRegionSilentlyAndWait(admin, rse.hsa, rse.hri);
1704         offline(rse.hri.getRegionName());
1705       } catch (IOException ioe) {
1706         LOG.warn("Got exception when attempting to offline region "
1707             + Bytes.toString(rse.hri.getRegionName()), ioe);
1708       }
1709     }
1710   }
1711 
1712   /**
1713    * Attempts to undeploy a region from a region server based in information in
1714    * META.  Any operations that modify the file system should make sure that
1715    * its corresponding region is not deployed to prevent data races.
1716    *
1717    * A separate call is required to update the master in-memory region state
1718    * kept in the AssignementManager.  Because disable uses this state instead of
1719    * that found in META, we can't seem to cleanly disable/delete tables that
1720    * have been hbck fixed.  When used on a version of HBase that does not have
1721    * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master
1722    * restart or failover may be required.
1723    */
1724   private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
1725     if (hi.metaEntry == null && hi.hdfsEntry == null) {
1726       undeployRegions(hi);
1727       return;
1728     }
1729 
1730     // get assignment info and hregioninfo from meta.
1731     Get get = new Get(hi.getRegionName());
1732     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1733     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
1734     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
1735     // also get the locations of the replicas to close if the primary region is being closed
1736     if (hi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1737       int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication();
1738       for (int i = 0; i < numReplicas; i++) {
1739         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
1740         get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(i));
1741       }
1742     }
1743     Result r = meta.get(get);
1744     RegionLocations rl = MetaTableAccessor.getRegionLocations(r);
1745     if (rl == null) {
1746       LOG.warn("Unable to close region " + hi.getRegionNameAsString() +
1747           " since meta does not have handle to reach it");
1748       return;
1749     }
1750     for (HRegionLocation h : rl.getRegionLocations()) {
1751       ServerName serverName = h.getServerName();
1752       if (serverName == null) {
1753         errors.reportError("Unable to close region "
1754             + hi.getRegionNameAsString() +  " because meta does not "
1755             + "have handle to reach it.");
1756         continue;
1757       }
1758       HRegionInfo hri = h.getRegionInfo();
1759       if (hri == null) {
1760         LOG.warn("Unable to close region " + hi.getRegionNameAsString()
1761             + " because hbase:meta had invalid or missing "
1762             + HConstants.CATALOG_FAMILY_STR + ":"
1763             + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
1764             + " qualifier value.");
1765         continue;
1766       }
1767       // close the region -- close files and remove assignment
1768       HBaseFsckRepair.closeRegionSilentlyAndWait(admin, serverName, hri);
1769     }
1770   }
1771 
1772   private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
1773     KeeperException, InterruptedException {
1774     // If we are trying to fix the errors
1775     if (shouldFixAssignments()) {
1776       errors.print(msg);
1777       undeployRegions(hbi);
1778       setShouldRerun();
1779       HRegionInfo hri = hbi.getHdfsHRI();
1780       if (hri == null) {
1781         hri = hbi.metaEntry;
1782       }
1783       HBaseFsckRepair.fixUnassigned(admin, hri);
1784       HBaseFsckRepair.waitUntilAssigned(admin, hri);
1785 
1786       // also assign replicas if needed (do it only when this call operates on a primary replica)
1787       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) return;
1788       int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication();
1789       for (int i = 1; i < replicationCount; i++) {
1790         hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
1791         HbckInfo h = regionInfoMap.get(hri.getEncodedName());
1792         if (h != null) {
1793           undeployRegions(h);
1794           //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore
1795           //in consistency checks
1796           h.setSkipChecks(true);
1797         }
1798         HBaseFsckRepair.fixUnassigned(admin, hri);
1799         HBaseFsckRepair.waitUntilAssigned(admin, hri);
1800       }
1801 
1802     }
1803   }
1804 
1805   /**
1806    * Check a single region for consistency and correct deployment.
1807    */
1808   private void checkRegionConsistency(final String key, final HbckInfo hbi)
1809   throws IOException, KeeperException, InterruptedException {
1810 
1811 	if (hbi.isSkipChecks()) return;
1812 	String descriptiveName = hbi.toString();
1813     boolean inMeta = hbi.metaEntry != null;
1814     // In case not checking HDFS, assume the region is on HDFS
1815     boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
1816     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
1817     boolean isDeployed = !hbi.deployedOn.isEmpty();
1818     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
1819     boolean deploymentMatchesMeta =
1820       hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
1821       hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
1822     boolean splitParent =
1823       (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
1824     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
1825     boolean recentlyModified = inHdfs &&
1826       hbi.getModTime() + timelag > System.currentTimeMillis();
1827 
1828     // ========== First the healthy cases =============
1829     if (hbi.containsOnlyHdfsEdits()) {
1830       return;
1831     }
1832     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
1833       return;
1834     } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
1835       LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
1836         "tabled that is not deployed");
1837       return;
1838     } else if (recentlyModified) {
1839       LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
1840       return;
1841     }
1842     // ========== Cases where the region is not in hbase:meta =============
1843     else if (!inMeta && !inHdfs && !isDeployed) {
1844       // We shouldn't have record of this region at all then!
1845       assert false : "Entry for region with no data";
1846     } else if (!inMeta && !inHdfs && isDeployed) {
1847       errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
1848           + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
1849           "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1850       if (shouldFixAssignments()) {
1851         undeployRegions(hbi);
1852       }
1853 
1854     } else if (!inMeta && inHdfs && !isDeployed) {
1855       if (hbi.isMerged()) {
1856         // This region has already been merged, the remaining hdfs file will be
1857         // cleaned by CatalogJanitor later
1858         hbi.setSkipChecks(true);
1859         LOG.info("Region " + descriptiveName
1860             + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
1861         return;
1862       }
1863       errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
1864           + descriptiveName + " on HDFS, but not listed in hbase:meta " +
1865           "or deployed on any region server");
1866       // restore region consistency of an adopted orphan
1867       if (shouldFixMeta()) {
1868         if (!hbi.isHdfsRegioninfoPresent()) {
1869           LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
1870               +  " in table integrity repair phase if -fixHdfsOrphans was" +
1871               " used.");
1872           return;
1873         }
1874 
1875         LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
1876         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
1877         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
1878             admin.getClusterStatus().getServers(), numReplicas);
1879 
1880         tryAssignmentRepair(hbi, "Trying to reassign region...");
1881       }
1882 
1883     } else if (!inMeta && inHdfs && isDeployed) {
1884       errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
1885           + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1886       debugLsr(hbi.getHdfsRegionDir());
1887       if (hbi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1888         // for replicas, this means that we should undeploy the region (we would have
1889         // gone over the primaries and fixed meta holes in first phase under
1890         // checkAndFixConsistency; we shouldn't get the condition !inMeta at
1891         // this stage unless unwanted replica)
1892         if (shouldFixAssignments()) {
1893           undeployRegionsForHbi(hbi);
1894         }
1895       }
1896       if (shouldFixMeta() && hbi.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1897         if (!hbi.isHdfsRegioninfoPresent()) {
1898           LOG.error("This should have been repaired in table integrity repair phase");
1899           return;
1900         }
1901 
1902         LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
1903         int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication();
1904         HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),
1905             admin.getClusterStatus().getServers(), numReplicas);
1906         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
1907       }
1908 
1909     // ========== Cases where the region is in hbase:meta =============
1910     } else if (inMeta && inHdfs && !isDeployed && splitParent) {
1911       // check whether this is an actual error, or just transient state where parent
1912       // is not cleaned
1913       if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
1914         // check that split daughters are there
1915         HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
1916         HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
1917         if (infoA != null && infoB != null) {
1918           // we already processed or will process daughters. Move on, nothing to see here.
1919           hbi.setSkipChecks(true);
1920           return;
1921         }
1922       }
1923       errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
1924           + descriptiveName + " is a split parent in META, in HDFS, "
1925           + "and not deployed on any region server. This could be transient.");
1926       if (shouldFixSplitParents()) {
1927         setShouldRerun();
1928         resetSplitParent(hbi);
1929       }
1930     } else if (inMeta && !inHdfs && !isDeployed) {
1931       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
1932           + descriptiveName + " found in META, but not in HDFS "
1933           + "or deployed on any region server.");
1934       if (shouldFixMeta()) {
1935         deleteMetaRegion(hbi);
1936       }
1937     } else if (inMeta && !inHdfs && isDeployed) {
1938       errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
1939           + " found in META, but not in HDFS, " +
1940           "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1941       // We treat HDFS as ground truth.  Any information in meta is transient
1942       // and equivalent data can be regenerated.  So, lets unassign and remove
1943       // these problems from META.
1944       if (shouldFixAssignments()) {
1945         errors.print("Trying to fix unassigned region...");
1946         undeployRegions(hbi);
1947       }
1948       if (shouldFixMeta()) {
1949         // wait for it to complete
1950         deleteMetaRegion(hbi);
1951       }
1952     } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
1953       errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
1954           + " not deployed on any region server.");
1955       tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
1956     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
1957       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
1958           "Region " + descriptiveName + " should not be deployed according " +
1959           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1960       if (shouldFixAssignments()) {
1961         errors.print("Trying to close the region " + descriptiveName);
1962         setShouldRerun();
1963         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
1964       }
1965     } else if (inMeta && inHdfs && isMultiplyDeployed) {
1966       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
1967           + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
1968           + " but is multiply assigned to region servers " +
1969           Joiner.on(", ").join(hbi.deployedOn));
1970       // If we are trying to fix the errors
1971       if (shouldFixAssignments()) {
1972         errors.print("Trying to fix assignment error...");
1973         setShouldRerun();
1974         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
1975       }
1976     } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
1977       errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
1978           + descriptiveName + " listed in hbase:meta on region server " +
1979           hbi.metaEntry.regionServer + " but found on region server " +
1980           hbi.deployedOn.get(0));
1981       // If we are trying to fix the errors
1982       if (shouldFixAssignments()) {
1983         errors.print("Trying to fix assignment error...");
1984         setShouldRerun();
1985         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
1986         HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
1987       }
1988     } else {
1989       errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
1990           " is in an unforeseen state:" +
1991           " inMeta=" + inMeta +
1992           " inHdfs=" + inHdfs +
1993           " isDeployed=" + isDeployed +
1994           " isMultiplyDeployed=" + isMultiplyDeployed +
1995           " deploymentMatchesMeta=" + deploymentMatchesMeta +
1996           " shouldBeDeployed=" + shouldBeDeployed);
1997     }
1998   }
1999 
2000   /**
2001    * Checks tables integrity. Goes over all regions and scans the tables.
2002    * Collects all the pieces for each table and checks if there are missing,
2003    * repeated or overlapping ones.
2004    * @throws IOException
2005    */
2006   SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
2007     tablesInfo = new TreeMap<TableName,TableInfo> ();
2008     LOG.debug("There are " + regionInfoMap.size() + " region info entries");
2009     for (HbckInfo hbi : regionInfoMap.values()) {
2010       // Check only valid, working regions
2011       if (hbi.metaEntry == null) {
2012         // this assumes that consistency check has run loadMetaEntry
2013         Path p = hbi.getHdfsRegionDir();
2014         if (p == null) {
2015           errors.report("No regioninfo in Meta or HDFS. " + hbi);
2016         }
2017 
2018         // TODO test.
2019         continue;
2020       }
2021       if (hbi.metaEntry.regionServer == null) {
2022         errors.detail("Skipping region because no region server: " + hbi);
2023         continue;
2024       }
2025       if (hbi.metaEntry.isOffline()) {
2026         errors.detail("Skipping region because it is offline: " + hbi);
2027         continue;
2028       }
2029       if (hbi.containsOnlyHdfsEdits()) {
2030         errors.detail("Skipping region because it only contains edits" + hbi);
2031         continue;
2032       }
2033 
2034       // Missing regionDir or over-deployment is checked elsewhere. Include
2035       // these cases in modTInfo, so we can evaluate those regions as part of
2036       // the region chain in META
2037       //if (hbi.foundRegionDir == null) continue;
2038       //if (hbi.deployedOn.size() != 1) continue;
2039       if (hbi.deployedOn.size() == 0) continue;
2040 
2041       // We should be safe here
2042       TableName tableName = hbi.metaEntry.getTable();
2043       TableInfo modTInfo = tablesInfo.get(tableName);
2044       if (modTInfo == null) {
2045         modTInfo = new TableInfo(tableName);
2046       }
2047       for (ServerName server : hbi.deployedOn) {
2048         modTInfo.addServer(server);
2049       }
2050 
2051       if (!hbi.isSkipChecks()) {
2052         modTInfo.addRegionInfo(hbi);
2053       }
2054 
2055       tablesInfo.put(tableName, modTInfo);
2056     }
2057 
2058     loadTableInfosForTablesWithNoRegion();
2059 
2060     for (TableInfo tInfo : tablesInfo.values()) {
2061       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
2062       if (!tInfo.checkRegionChain(handler)) {
2063         errors.report("Found inconsistency in table " + tInfo.getName());
2064       }
2065     }
2066     return tablesInfo;
2067   }
2068 
2069   /** Loads table info's for tables that may not have been included, since there are no
2070    * regions reported for the table, but table dir is there in hdfs
2071    */
2072   private void loadTableInfosForTablesWithNoRegion() throws IOException {
2073     Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
2074     for (HTableDescriptor htd : allTables.values()) {
2075       if (checkMetaOnly && !htd.isMetaTable()) {
2076         continue;
2077       }
2078 
2079       TableName tableName = htd.getTableName();
2080       if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2081         TableInfo tableInfo = new TableInfo(tableName);
2082         tableInfo.htds.add(htd);
2083         tablesInfo.put(htd.getTableName(), tableInfo);
2084       }
2085     }
2086   }
2087 
2088   /**
2089    * Merge hdfs data by moving from contained HbckInfo into targetRegionDir.
2090    * @return number of file move fixes done to merge regions.
2091    */
2092   public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2093     int fileMoves = 0;
2094     String thread = Thread.currentThread().getName();
2095     LOG.debug("[" + thread + "] Contained region dir after close and pause");
2096     debugLsr(contained.getHdfsRegionDir());
2097 
2098     // rename the contained into the container.
2099     FileSystem fs = targetRegionDir.getFileSystem(getConf());
2100     FileStatus[] dirs = null;
2101     try {
2102       dirs = fs.listStatus(contained.getHdfsRegionDir());
2103     } catch (FileNotFoundException fnfe) {
2104       // region we are attempting to merge in is not present!  Since this is a merge, there is
2105       // no harm skipping this region if it does not exist.
2106       if (!fs.exists(contained.getHdfsRegionDir())) {
2107         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2108             + " is missing. Assuming already sidelined or moved.");
2109       } else {
2110         sidelineRegionDir(fs, contained);
2111       }
2112       return fileMoves;
2113     }
2114 
2115     if (dirs == null) {
2116       if (!fs.exists(contained.getHdfsRegionDir())) {
2117         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2118             + " already sidelined.");
2119       } else {
2120         sidelineRegionDir(fs, contained);
2121       }
2122       return fileMoves;
2123     }
2124 
2125     for (FileStatus cf : dirs) {
2126       Path src = cf.getPath();
2127       Path dst =  new Path(targetRegionDir, src.getName());
2128 
2129       if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2130         // do not copy the old .regioninfo file.
2131         continue;
2132       }
2133 
2134       if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2135         // do not copy the .oldlogs files
2136         continue;
2137       }
2138 
2139       LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2140       // FileSystem.rename is inconsistent with directories -- if the
2141       // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2142       // it moves the src into the dst dir resulting in (foo/a/b).  If
2143       // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2144       for (FileStatus hfile : fs.listStatus(src)) {
2145         boolean success = fs.rename(hfile.getPath(), dst);
2146         if (success) {
2147           fileMoves++;
2148         }
2149       }
2150       LOG.debug("[" + thread + "] Sideline directory contents:");
2151       debugLsr(targetRegionDir);
2152     }
2153 
2154     // if all success.
2155     sidelineRegionDir(fs, contained);
2156     LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2157         getSidelineDir());
2158     debugLsr(contained.getHdfsRegionDir());
2159 
2160     return fileMoves;
2161   }
2162 
2163 
2164   static class WorkItemOverlapMerge implements Callable<Void> {
2165     private TableIntegrityErrorHandler handler;
2166     Collection<HbckInfo> overlapgroup;
2167 
2168     WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2169       this.handler = handler;
2170       this.overlapgroup = overlapgroup;
2171     }
2172 
2173     @Override
2174     public Void call() throws Exception {
2175       handler.handleOverlapGroup(overlapgroup);
2176       return null;
2177     }
2178   };
2179 
2180 
2181   /**
2182    * Maintain information about a particular table.
2183    */
2184   public class TableInfo {
2185     TableName tableName;
2186     TreeSet <ServerName> deployedOn;
2187 
2188     // backwards regions
2189     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2190 
2191     // sidelined big overlapped regions
2192     final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2193 
2194     // region split calculator
2195     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2196 
2197     // Histogram of different HTableDescriptors found.  Ideally there is only one!
2198     final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2199 
2200     // key = start split, values = set of splits in problem group
2201     final Multimap<byte[], HbckInfo> overlapGroups =
2202       TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2203 
2204     TableInfo(TableName name) {
2205       this.tableName = name;
2206       deployedOn = new TreeSet <ServerName>();
2207     }
2208 
2209     /**
2210      * @return descriptor common to all regions.  null if are none or multiple!
2211      */
2212     private HTableDescriptor getHTD() {
2213       if (htds.size() == 1) {
2214         return (HTableDescriptor)htds.toArray()[0];
2215       } else {
2216         LOG.error("None/Multiple table descriptors found for table '"
2217           + tableName + "' regions: " + htds);
2218       }
2219       return null;
2220     }
2221 
2222     public void addRegionInfo(HbckInfo hir) {
2223       if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2224         // end key is absolute end key, just add it.
2225         // ignore replicas other than primary for these checks
2226         if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2227         return;
2228       }
2229 
2230       // if not the absolute end key, check for cycle
2231       if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2232         errors.reportError(
2233             ERROR_CODE.REGION_CYCLE,
2234             String.format("The endkey for this region comes before the "
2235                 + "startkey, startkey=%s, endkey=%s",
2236                 Bytes.toStringBinary(hir.getStartKey()),
2237                 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2238         backwards.add(hir);
2239         return;
2240       }
2241 
2242       // main case, add to split calculator
2243       // ignore replicas other than primary for these checks
2244       if (hir.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) sc.add(hir);
2245     }
2246 
2247     public void addServer(ServerName server) {
2248       this.deployedOn.add(server);
2249     }
2250 
2251     public TableName getName() {
2252       return tableName;
2253     }
2254 
2255     public int getNumRegions() {
2256       return sc.getStarts().size() + backwards.size();
2257     }
2258 
2259     private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2260       ErrorReporter errors;
2261 
2262       IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2263         this.errors = errors;
2264         setTableInfo(ti);
2265       }
2266 
2267       @Override
2268       public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2269         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2270             "First region should start with an empty key.  You need to "
2271             + " create a new region and regioninfo in HDFS to plug the hole.",
2272             getTableInfo(), hi);
2273       }
2274 
2275       @Override
2276       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2277         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2278             "Last region should end with an empty key. You need to "
2279                 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2280       }
2281 
2282       @Override
2283       public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2284         errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2285             "Region has the same start and end key.", getTableInfo(), hi);
2286       }
2287 
2288       @Override
2289       public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2290         byte[] key = r1.getStartKey();
2291         // dup start key
2292         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2293             "Multiple regions have the same startkey: "
2294             + Bytes.toStringBinary(key), getTableInfo(), r1);
2295         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2296             "Multiple regions have the same startkey: "
2297             + Bytes.toStringBinary(key), getTableInfo(), r2);
2298       }
2299 
2300       @Override
2301       public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2302         errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2303             "There is an overlap in the region chain.",
2304             getTableInfo(), hi1, hi2);
2305       }
2306 
2307       @Override
2308       public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2309         errors.reportError(
2310             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2311             "There is a hole in the region chain between "
2312                 + Bytes.toStringBinary(holeStart) + " and "
2313                 + Bytes.toStringBinary(holeStop)
2314                 + ".  You need to create a new .regioninfo and region "
2315                 + "dir in hdfs to plug the hole.");
2316       }
2317     };
2318 
2319     /**
2320      * This handler fixes integrity errors from hdfs information.  There are
2321      * basically three classes of integrity problems 1) holes, 2) overlaps, and
2322      * 3) invalid regions.
2323      *
2324      * This class overrides methods that fix holes and the overlap group case.
2325      * Individual cases of particular overlaps are handled by the general
2326      * overlap group merge repair case.
2327      *
2328      * If hbase is online, this forces regions offline before doing merge
2329      * operations.
2330      */
2331     private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2332       Configuration conf;
2333 
2334       boolean fixOverlaps = true;
2335 
2336       HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2337           boolean fixHoles, boolean fixOverlaps) {
2338         super(ti, errors);
2339         this.conf = conf;
2340         this.fixOverlaps = fixOverlaps;
2341         // TODO properly use fixHoles
2342       }
2343 
2344       /**
2345        * This is a special case hole -- when the first region of a table is
2346        * missing from META, HBase doesn't acknowledge the existance of the
2347        * table.
2348        */
2349       @Override
2350       public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2351         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2352             "First region should start with an empty key.  Creating a new " +
2353             "region and regioninfo in HDFS to plug the hole.",
2354             getTableInfo(), next);
2355         HTableDescriptor htd = getTableInfo().getHTD();
2356         // from special EMPTY_START_ROW to next region's startKey
2357         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2358             HConstants.EMPTY_START_ROW, next.getStartKey());
2359 
2360         // TODO test
2361         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2362         LOG.info("Table region start key was not empty.  Created new empty region: "
2363             + newRegion + " " +region);
2364         fixes++;
2365       }
2366 
2367       @Override
2368       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2369         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2370             "Last region should end with an empty key.  Creating a new "
2371                 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2372         HTableDescriptor htd = getTableInfo().getHTD();
2373         // from curEndKey to EMPTY_START_ROW
2374         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2375             HConstants.EMPTY_START_ROW);
2376 
2377         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2378         LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
2379             + " " + region);
2380         fixes++;
2381       }
2382 
2383       /**
2384        * There is a hole in the hdfs regions that violates the table integrity
2385        * rules.  Create a new empty region that patches the hole.
2386        */
2387       @Override
2388       public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2389         errors.reportError(
2390             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2391             "There is a hole in the region chain between "
2392                 + Bytes.toStringBinary(holeStartKey) + " and "
2393                 + Bytes.toStringBinary(holeStopKey)
2394                 + ".  Creating a new regioninfo and region "
2395                 + "dir in hdfs to plug the hole.");
2396         HTableDescriptor htd = getTableInfo().getHTD();
2397         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2398         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2399         LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2400         fixes++;
2401       }
2402 
2403       /**
2404        * This takes set of overlapping regions and merges them into a single
2405        * region.  This covers cases like degenerate regions, shared start key,
2406        * general overlaps, duplicate ranges, and partial overlapping regions.
2407        *
2408        * Cases:
2409        * - Clean regions that overlap
2410        * - Only .oldlogs regions (can't find start/stop range, or figure out)
2411        *
2412        * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
2413        */
2414       @Override
2415       public void handleOverlapGroup(Collection<HbckInfo> overlap)
2416           throws IOException {
2417         Preconditions.checkNotNull(overlap);
2418         Preconditions.checkArgument(overlap.size() >0);
2419 
2420         if (!this.fixOverlaps) {
2421           LOG.warn("Not attempting to repair overlaps.");
2422           return;
2423         }
2424 
2425         if (overlap.size() > maxMerge) {
2426           LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2427             "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2428           if (sidelineBigOverlaps) {
2429             // we only sideline big overlapped groups that exceeds the max number of regions to merge
2430             sidelineBigOverlaps(overlap);
2431           }
2432           return;
2433         }
2434 
2435         mergeOverlaps(overlap);
2436       }
2437 
2438       void mergeOverlaps(Collection<HbckInfo> overlap)
2439           throws IOException {
2440         String thread = Thread.currentThread().getName();
2441         LOG.info("== [" + thread + "] Merging regions into one region: "
2442           + Joiner.on(",").join(overlap));
2443         // get the min / max range and close all concerned regions
2444         Pair<byte[], byte[]> range = null;
2445         for (HbckInfo hi : overlap) {
2446           if (range == null) {
2447             range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2448           } else {
2449             if (RegionSplitCalculator.BYTES_COMPARATOR
2450                 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2451               range.setFirst(hi.getStartKey());
2452             }
2453             if (RegionSplitCalculator.BYTES_COMPARATOR
2454                 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2455               range.setSecond(hi.getEndKey());
2456             }
2457           }
2458           // need to close files so delete can happen.
2459           LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
2460           LOG.debug("[" + thread + "] Contained region dir before close");
2461           debugLsr(hi.getHdfsRegionDir());
2462           try {
2463             LOG.info("[" + thread + "] Closing region: " + hi);
2464             closeRegion(hi);
2465           } catch (IOException ioe) {
2466             LOG.warn("[" + thread + "] Was unable to close region " + hi
2467               + ".  Just continuing... ", ioe);
2468           } catch (InterruptedException e) {
2469             LOG.warn("[" + thread + "] Was unable to close region " + hi
2470               + ".  Just continuing... ", e);
2471           }
2472 
2473           try {
2474             LOG.info("[" + thread + "] Offlining region: " + hi);
2475             offline(hi.getRegionName());
2476           } catch (IOException ioe) {
2477             LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2478               + ".  Just continuing... ", ioe);
2479           }
2480         }
2481 
2482         // create new empty container region.
2483         HTableDescriptor htd = getTableInfo().getHTD();
2484         // from start key to end Key
2485         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
2486             range.getSecond());
2487         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2488         LOG.info("[" + thread + "] Created new empty container region: " +
2489             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
2490         debugLsr(region.getRegionFileSystem().getRegionDir());
2491 
2492         // all target regions are closed, should be able to safely cleanup.
2493         boolean didFix= false;
2494         Path target = region.getRegionFileSystem().getRegionDir();
2495         for (HbckInfo contained : overlap) {
2496           LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
2497           int merges = mergeRegionDirs(target, contained);
2498           if (merges > 0) {
2499             didFix = true;
2500           }
2501         }
2502         if (didFix) {
2503           fixes++;
2504         }
2505       }
2506 
2507       /**
2508        * Sideline some regions in a big overlap group so that it
2509        * will have fewer regions, and it is easier to merge them later on.
2510        *
2511        * @param bigOverlap the overlapped group with regions more than maxMerge
2512        * @throws IOException
2513        */
2514       void sidelineBigOverlaps(
2515           Collection<HbckInfo> bigOverlap) throws IOException {
2516         int overlapsToSideline = bigOverlap.size() - maxMerge;
2517         if (overlapsToSideline > maxOverlapsToSideline) {
2518           overlapsToSideline = maxOverlapsToSideline;
2519         }
2520         List<HbckInfo> regionsToSideline =
2521           RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
2522         FileSystem fs = FileSystem.get(conf);
2523         for (HbckInfo regionToSideline: regionsToSideline) {
2524           try {
2525             LOG.info("Closing region: " + regionToSideline);
2526             closeRegion(regionToSideline);
2527           } catch (IOException ioe) {
2528             LOG.warn("Was unable to close region " + regionToSideline
2529               + ".  Just continuing... ", ioe);
2530           } catch (InterruptedException e) {
2531             LOG.warn("Was unable to close region " + regionToSideline
2532               + ".  Just continuing... ", e);
2533           }
2534 
2535           try {
2536             LOG.info("Offlining region: " + regionToSideline);
2537             offline(regionToSideline.getRegionName());
2538           } catch (IOException ioe) {
2539             LOG.warn("Unable to offline region from master: " + regionToSideline
2540               + ".  Just continuing... ", ioe);
2541           }
2542 
2543           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
2544           Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
2545           if (sidelineRegionDir != null) {
2546             sidelinedRegions.put(sidelineRegionDir, regionToSideline);
2547             LOG.info("After sidelined big overlapped region: "
2548               + regionToSideline.getRegionNameAsString()
2549               + " to " + sidelineRegionDir.toString());
2550             fixes++;
2551           }
2552         }
2553       }
2554     }
2555 
2556     /**
2557      * Check the region chain (from META) of this table.  We are looking for
2558      * holes, overlaps, and cycles.
2559      * @return false if there are errors
2560      * @throws IOException
2561      */
2562     public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
2563       // When table is disabled no need to check for the region chain. Some of the regions
2564       // accidently if deployed, this below code might report some issues like missing start
2565       // or end regions or region hole in chain and may try to fix which is unwanted.
2566       if (disabledTables.contains(this.tableName)) {
2567         return true;
2568       }
2569       int originalErrorsCount = errors.getErrorList().size();
2570       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
2571       SortedSet<byte[]> splits = sc.getSplits();
2572 
2573       byte[] prevKey = null;
2574       byte[] problemKey = null;
2575 
2576       if (splits.size() == 0) {
2577         // no region for this table
2578         handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
2579       }
2580 
2581       for (byte[] key : splits) {
2582         Collection<HbckInfo> ranges = regions.get(key);
2583         if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
2584           for (HbckInfo rng : ranges) {
2585             handler.handleRegionStartKeyNotEmpty(rng);
2586           }
2587         }
2588 
2589         // check for degenerate ranges
2590         for (HbckInfo rng : ranges) {
2591           // special endkey case converts '' to null
2592           byte[] endKey = rng.getEndKey();
2593           endKey = (endKey.length == 0) ? null : endKey;
2594           if (Bytes.equals(rng.getStartKey(),endKey)) {
2595             handler.handleDegenerateRegion(rng);
2596           }
2597         }
2598 
2599         if (ranges.size() == 1) {
2600           // this split key is ok -- no overlap, not a hole.
2601           if (problemKey != null) {
2602             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2603           }
2604           problemKey = null; // fell through, no more problem.
2605         } else if (ranges.size() > 1) {
2606           // set the new problem key group name, if already have problem key, just
2607           // keep using it.
2608           if (problemKey == null) {
2609             // only for overlap regions.
2610             LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
2611             problemKey = key;
2612           }
2613           overlapGroups.putAll(problemKey, ranges);
2614 
2615           // record errors
2616           ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
2617           //  this dumb and n^2 but this shouldn't happen often
2618           for (HbckInfo r1 : ranges) {
2619             if (r1.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2620             subRange.remove(r1);
2621             for (HbckInfo r2 : subRange) {
2622               if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue;
2623               if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
2624                 handler.handleDuplicateStartKeys(r1,r2);
2625               } else {
2626                 // overlap
2627                 handler.handleOverlapInRegionChain(r1, r2);
2628               }
2629             }
2630           }
2631 
2632         } else if (ranges.size() == 0) {
2633           if (problemKey != null) {
2634             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2635           }
2636           problemKey = null;
2637 
2638           byte[] holeStopKey = sc.getSplits().higher(key);
2639           // if higher key is null we reached the top.
2640           if (holeStopKey != null) {
2641             // hole
2642             handler.handleHoleInRegionChain(key, holeStopKey);
2643           }
2644         }
2645         prevKey = key;
2646       }
2647 
2648       // When the last region of a table is proper and having an empty end key, 'prevKey'
2649       // will be null.
2650       if (prevKey != null) {
2651         handler.handleRegionEndKeyNotEmpty(prevKey);
2652       }
2653 
2654       // TODO fold this into the TableIntegrityHandler
2655       if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
2656         LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
2657             " false to run serially.");
2658         boolean ok = handleOverlapsParallel(handler, prevKey);
2659         if (!ok) {
2660           return false;
2661         }
2662       } else {
2663         LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
2664             " true to run in parallel.");
2665         for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2666           handler.handleOverlapGroup(overlap);
2667         }
2668       }
2669 
2670       if (details) {
2671         // do full region split map dump
2672         errors.print("---- Table '"  +  this.tableName
2673             + "': region split map");
2674         dump(splits, regions);
2675         errors.print("---- Table '"  +  this.tableName
2676             + "': overlap groups");
2677         dumpOverlapProblems(overlapGroups);
2678         errors.print("There are " + overlapGroups.keySet().size()
2679             + " overlap groups with " + overlapGroups.size()
2680             + " overlapping regions");
2681       }
2682       if (!sidelinedRegions.isEmpty()) {
2683         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
2684         errors.print("---- Table '"  +  this.tableName
2685             + "': sidelined big overlapped regions");
2686         dumpSidelinedRegions(sidelinedRegions);
2687       }
2688       return errors.getErrorList().size() == originalErrorsCount;
2689     }
2690 
2691     private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
2692         throws IOException {
2693       // we parallelize overlap handler for the case we have lots of groups to fix.  We can
2694       // safely assume each group is independent.
2695       List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
2696       List<Future<Void>> rets;
2697       for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2698         //
2699         merges.add(new WorkItemOverlapMerge(overlap, handler));
2700       }
2701       try {
2702         rets = executor.invokeAll(merges);
2703       } catch (InterruptedException e) {
2704         LOG.error("Overlap merges were interrupted", e);
2705         return false;
2706       }
2707       for(int i=0; i<merges.size(); i++) {
2708         WorkItemOverlapMerge work = merges.get(i);
2709         Future<Void> f = rets.get(i);
2710         try {
2711           f.get();
2712         } catch(ExecutionException e) {
2713           LOG.warn("Failed to merge overlap group" + work, e.getCause());
2714         } catch (InterruptedException e) {
2715           LOG.error("Waiting for overlap merges was interrupted", e);
2716           return false;
2717         }
2718       }
2719       return true;
2720     }
2721 
2722     /**
2723      * This dumps data in a visually reasonable way for visual debugging
2724      *
2725      * @param splits
2726      * @param regions
2727      */
2728     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
2729       // we display this way because the last end key should be displayed as well.
2730       StringBuilder sb = new StringBuilder();
2731       for (byte[] k : splits) {
2732         sb.setLength(0); // clear out existing buffer, if any.
2733         sb.append(Bytes.toStringBinary(k) + ":\t");
2734         for (HbckInfo r : regions.get(k)) {
2735           sb.append("[ "+ r.toString() + ", "
2736               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
2737         }
2738         errors.print(sb.toString());
2739       }
2740     }
2741   }
2742 
2743   public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
2744     // we display this way because the last end key should be displayed as
2745     // well.
2746     for (byte[] k : regions.keySet()) {
2747       errors.print(Bytes.toStringBinary(k) + ":");
2748       for (HbckInfo r : regions.get(k)) {
2749         errors.print("[ " + r.toString() + ", "
2750             + Bytes.toStringBinary(r.getEndKey()) + "]");
2751       }
2752       errors.print("----");
2753     }
2754   }
2755 
2756   public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
2757     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
2758       TableName tableName = entry.getValue().getTableName();
2759       Path path = entry.getKey();
2760       errors.print("This sidelined region dir should be bulk loaded: "
2761         + path.toString());
2762       errors.print("Bulk load command looks like: "
2763         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
2764         + path.toUri().getPath() + " "+ tableName);
2765     }
2766   }
2767 
2768   public Multimap<byte[], HbckInfo> getOverlapGroups(
2769       TableName table) {
2770     TableInfo ti = tablesInfo.get(table);
2771     return ti.overlapGroups;
2772   }
2773 
2774   /**
2775    * Return a list of user-space table names whose metadata have not been
2776    * modified in the last few milliseconds specified by timelag
2777    * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
2778    * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
2779    * milliseconds specified by timelag, then the table is a candidate to be returned.
2780    * @return tables that have not been modified recently
2781    * @throws IOException if an error is encountered
2782    */
2783   HTableDescriptor[] getTables(AtomicInteger numSkipped) {
2784     List<TableName> tableNames = new ArrayList<TableName>();
2785     long now = System.currentTimeMillis();
2786 
2787     for (HbckInfo hbi : regionInfoMap.values()) {
2788       MetaEntry info = hbi.metaEntry;
2789 
2790       // if the start key is zero, then we have found the first region of a table.
2791       // pick only those tables that were not modified in the last few milliseconds.
2792       if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
2793         if (info.modTime + timelag < now) {
2794           tableNames.add(info.getTable());
2795         } else {
2796           numSkipped.incrementAndGet(); // one more in-flux table
2797         }
2798       }
2799     }
2800     return getHTableDescriptors(tableNames);
2801   }
2802 
2803   HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
2804     HTableDescriptor[] htd = new HTableDescriptor[0];
2805     Admin admin = null;
2806     try {
2807       LOG.info("getHTableDescriptors == tableNames => " + tableNames);
2808       admin = new HBaseAdmin(getConf());
2809       htd = admin.getTableDescriptorsByTableName(tableNames);
2810     } catch (IOException e) {
2811       LOG.debug("Exception getting table descriptors", e);
2812     } finally {
2813       if (admin != null) {
2814         try {
2815           admin.close();
2816         } catch (IOException e) {
2817           LOG.debug("Exception closing HBaseAdmin", e);
2818         }
2819       }
2820     }
2821     return htd;
2822   }
2823 
2824   /**
2825    * Gets the entry in regionInfo corresponding to the the given encoded
2826    * region name. If the region has not been seen yet, a new entry is added
2827    * and returned.
2828    */
2829   private synchronized HbckInfo getOrCreateInfo(String name) {
2830     HbckInfo hbi = regionInfoMap.get(name);
2831     if (hbi == null) {
2832       hbi = new HbckInfo(null);
2833       regionInfoMap.put(name, hbi);
2834     }
2835     return hbi;
2836   }
2837 
2838   private void checkAndFixTableLocks() throws IOException {
2839     TableLockChecker checker = new TableLockChecker(createZooKeeperWatcher(), errors);
2840     checker.checkTableLocks();
2841 
2842     if (this.fixTableLocks) {
2843       checker.fixExpiredTableLocks();
2844     }
2845   }
2846 
2847   /**
2848     * Check values in regionInfo for hbase:meta
2849     * Check if zero or more than one regions with hbase:meta are found.
2850     * If there are inconsistencies (i.e. zero or more than one regions
2851     * pretend to be holding the hbase:meta) try to fix that and report an error.
2852     * @throws IOException from HBaseFsckRepair functions
2853    * @throws KeeperException
2854    * @throws InterruptedException
2855     */
2856   boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
2857     List<HbckInfo> metaRegions = Lists.newArrayList();
2858     for (HbckInfo value : regionInfoMap.values()) {
2859       if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
2860         metaRegions.add(value);
2861       }
2862     }
2863 
2864     // There will be always one entry in regionInfoMap corresponding to hbase:meta
2865     // Check the deployed servers. It should be exactly one server.
2866     List<ServerName> servers = new ArrayList<ServerName>();
2867     HbckInfo metaHbckInfo = null;
2868     if (!metaRegions.isEmpty()) {
2869       metaHbckInfo = metaRegions.get(0);
2870       servers = metaHbckInfo.deployedOn;
2871     }
2872     if (servers.size() != 1) {
2873       if (servers.size() == 0) {
2874         errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta is not found on any region.");
2875         if (shouldFixAssignments()) {
2876           errors.print("Trying to fix a problem with hbase:meta..");
2877           setShouldRerun();
2878           // try to fix it (treat it as unassigned region)
2879           HBaseFsckRepair.fixUnassigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
2880           HBaseFsckRepair.waitUntilAssigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
2881         }
2882       } else if (servers.size() > 1) {
2883         errors
2884             .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta is found on more than one region.");
2885         if (shouldFixAssignments()) {
2886           if (metaHbckInfo == null) {
2887             errors.print(
2888               "Unable to fix problem with hbase:meta due to hbase:meta region info missing");
2889             return false;
2890           }
2891           errors.print("Trying to fix a problem with hbase:meta..");
2892           setShouldRerun();
2893           // try fix it (treat is a dupe assignment)
2894           HBaseFsckRepair.fixMultiAssignment(admin, metaHbckInfo.metaEntry, servers);
2895         }
2896       }
2897       // rerun hbck with hopefully fixed META
2898       return false;
2899     }
2900     // no errors, so continue normally
2901     return true;
2902   }
2903 
2904   /**
2905    * Scan hbase:meta, adding all regions found to the regionInfo map.
2906    * @throws IOException if an error is encountered
2907    */
2908   boolean loadMetaEntries() throws IOException {
2909     MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
2910       int countRecord = 1;
2911 
2912       // comparator to sort KeyValues with latest modtime
2913       final Comparator<Cell> comp = new Comparator<Cell>() {
2914         @Override
2915         public int compare(Cell k1, Cell k2) {
2916           return (int)(k1.getTimestamp() - k2.getTimestamp());
2917         }
2918       };
2919 
2920       @Override
2921       public boolean processRow(Result result) throws IOException {
2922         try {
2923 
2924           // record the latest modification of this META record
2925           long ts =  Collections.max(result.listCells(), comp).getTimestamp();
2926           RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
2927           if (rl == null) {
2928             emptyRegionInfoQualifiers.add(result);
2929             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
2930               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
2931             return true;
2932           }
2933           ServerName sn = null;
2934           if (rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID) == null ||
2935               rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo() == null) {
2936             emptyRegionInfoQualifiers.add(result);
2937             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
2938               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
2939             return true;
2940           }
2941           HRegionInfo hri = rl.getRegionLocation(HRegionInfo.DEFAULT_REPLICA_ID).getRegionInfo();
2942           if (!(isTableIncluded(hri.getTable())
2943               || hri.isMetaRegion())) {
2944             return true;
2945           }
2946           PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
2947           for (HRegionLocation h : rl.getRegionLocations()) {
2948             if (h == null || h.getRegionInfo() == null) {
2949               continue;
2950             }
2951             sn = h.getServerName();
2952             hri = h.getRegionInfo();
2953 
2954             MetaEntry m = null;
2955             if (hri.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
2956               m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
2957             } else {
2958               m = new MetaEntry(hri, sn, ts, null, null);
2959             }
2960             HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
2961             if (previous == null) {
2962               regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
2963             } else if (previous.metaEntry == null) {
2964               previous.metaEntry = m;
2965             } else {
2966               throw new IOException("Two entries in hbase:meta are same " + previous);
2967             }
2968           }
2969           PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
2970           for (HRegionInfo mergeRegion : new HRegionInfo[] {
2971               mergeRegions.getFirst(), mergeRegions.getSecond() }) {
2972             if (mergeRegion != null) {
2973               // This region is already been merged
2974               HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
2975               hbInfo.setMerged(true);
2976             }
2977           }
2978 
2979           // show proof of progress to the user, once for every 100 records.
2980           if (countRecord % 100 == 0) {
2981             errors.progress();
2982           }
2983           countRecord++;
2984           return true;
2985         } catch (RuntimeException e) {
2986           LOG.error("Result=" + result);
2987           throw e;
2988         }
2989       }
2990     };
2991     if (!checkMetaOnly) {
2992       // Scan hbase:meta to pick up user regions
2993       MetaScanner.metaScan(getConf(), visitor);
2994     }
2995 
2996     errors.print("");
2997     return true;
2998   }
2999 
3000   /**
3001    * Stores the regioninfo entries scanned from META
3002    */
3003   static class MetaEntry extends HRegionInfo {
3004     ServerName regionServer;   // server hosting this region
3005     long modTime;          // timestamp of most recent modification metadata
3006     HRegionInfo splitA, splitB; //split daughters
3007 
3008     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
3009       this(rinfo, regionServer, modTime, null, null);
3010     }
3011 
3012     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
3013         HRegionInfo splitA, HRegionInfo splitB) {
3014       super(rinfo);
3015       this.regionServer = regionServer;
3016       this.modTime = modTime;
3017       this.splitA = splitA;
3018       this.splitB = splitB;
3019     }
3020 
3021     @Override
3022     public boolean equals(Object o) {
3023       boolean superEq = super.equals(o);
3024       if (!superEq) {
3025         return superEq;
3026       }
3027 
3028       MetaEntry me = (MetaEntry) o;
3029       if (!regionServer.equals(me.regionServer)) {
3030         return false;
3031       }
3032       return (modTime == me.modTime);
3033     }
3034 
3035     @Override
3036     public int hashCode() {
3037       int hash = Arrays.hashCode(getRegionName());
3038       hash ^= getRegionId();
3039       hash ^= Arrays.hashCode(getStartKey());
3040       hash ^= Arrays.hashCode(getEndKey());
3041       hash ^= Boolean.valueOf(isOffline()).hashCode();
3042       hash ^= getTable().hashCode();
3043       if (regionServer != null) {
3044         hash ^= regionServer.hashCode();
3045       }
3046       hash ^= modTime;
3047       return hash;
3048     }
3049   }
3050 
3051   /**
3052    * Stores the regioninfo entries from HDFS
3053    */
3054   static class HdfsEntry {
3055     HRegionInfo hri;
3056     Path hdfsRegionDir = null;
3057     long hdfsRegionDirModTime  = 0;
3058     boolean hdfsRegioninfoFilePresent = false;
3059     boolean hdfsOnlyEdits = false;
3060   }
3061 
3062   /**
3063    * Stores the regioninfo retrieved from Online region servers.
3064    */
3065   static class OnlineEntry {
3066     HRegionInfo hri;
3067     ServerName hsa;
3068 
3069     @Override
3070     public String toString() {
3071       return hsa.toString() + ";" + hri.getRegionNameAsString();
3072     }
3073   }
3074 
3075   /**
3076    * Maintain information about a particular region.  It gathers information
3077    * from three places -- HDFS, META, and region servers.
3078    */
3079   public static class HbckInfo implements KeyRange {
3080     private MetaEntry metaEntry = null; // info in META
3081     private HdfsEntry hdfsEntry = null; // info in HDFS
3082     private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
3083     private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
3084     private boolean skipChecks = false; // whether to skip further checks to this region info.
3085     private boolean isMerged = false;// whether this region has already been merged into another one
3086     private int deployedReplicaId = HRegionInfo.DEFAULT_REPLICA_ID;
3087     private HRegionInfo primaryHRIForDeployedReplica = null;
3088 
3089     HbckInfo(MetaEntry metaEntry) {
3090       this.metaEntry = metaEntry;
3091     }
3092 
3093     public int getReplicaId() {
3094       if (metaEntry != null) return metaEntry.getReplicaId();
3095       return deployedReplicaId;
3096     }
3097 
3098     public synchronized void addServer(HRegionInfo hri, ServerName server) {
3099       OnlineEntry rse = new OnlineEntry() ;
3100       rse.hri = hri;
3101       rse.hsa = server;
3102       this.deployedEntries.add(rse);
3103       this.deployedOn.add(server);
3104       // save the replicaId that we see deployed in the cluster
3105       this.deployedReplicaId = hri.getReplicaId();
3106       this.primaryHRIForDeployedReplica =
3107           RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
3108     }
3109 
3110     @Override
3111     public synchronized String toString() {
3112       StringBuilder sb = new StringBuilder();
3113       sb.append("{ meta => ");
3114       sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3115       sb.append( ", hdfs => " + getHdfsRegionDir());
3116       sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3117       sb.append( ", replicaId => " + getReplicaId());
3118       sb.append(" }");
3119       return sb.toString();
3120     }
3121 
3122     @Override
3123     public byte[] getStartKey() {
3124       if (this.metaEntry != null) {
3125         return this.metaEntry.getStartKey();
3126       } else if (this.hdfsEntry != null) {
3127         return this.hdfsEntry.hri.getStartKey();
3128       } else {
3129         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3130         return null;
3131       }
3132     }
3133 
3134     @Override
3135     public byte[] getEndKey() {
3136       if (this.metaEntry != null) {
3137         return this.metaEntry.getEndKey();
3138       } else if (this.hdfsEntry != null) {
3139         return this.hdfsEntry.hri.getEndKey();
3140       } else {
3141         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3142         return null;
3143       }
3144     }
3145 
3146     public TableName getTableName() {
3147       if (this.metaEntry != null) {
3148         return this.metaEntry.getTable();
3149       } else if (this.hdfsEntry != null) {
3150         // we are only guaranteed to have a path and not an HRI for hdfsEntry,
3151         // so we get the name from the Path
3152         Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3153         return FSUtils.getTableName(tableDir);
3154       } else {
3155         // return the info from the first online/deployed hri
3156         for (OnlineEntry e : deployedEntries) {
3157           return e.hri.getTable();
3158         }
3159         return null;
3160       }
3161     }
3162 
3163     public String getRegionNameAsString() {
3164       if (metaEntry != null) {
3165         return metaEntry.getRegionNameAsString();
3166       } else if (hdfsEntry != null) {
3167         if (hdfsEntry.hri != null) {
3168           return hdfsEntry.hri.getRegionNameAsString();
3169         }
3170       } else {
3171         // return the info from the first online/deployed hri
3172         for (OnlineEntry e : deployedEntries) {
3173           return e.hri.getRegionNameAsString();
3174         }
3175       }
3176       return null;
3177     }
3178 
3179     public byte[] getRegionName() {
3180       if (metaEntry != null) {
3181         return metaEntry.getRegionName();
3182       } else if (hdfsEntry != null) {
3183         return hdfsEntry.hri.getRegionName();
3184       } else {
3185         // return the info from the first online/deployed hri
3186         for (OnlineEntry e : deployedEntries) {
3187           return e.hri.getRegionName();
3188         }
3189         return null;
3190       }
3191     }
3192 
3193     public HRegionInfo getPrimaryHRIForDeployedReplica() {
3194       return primaryHRIForDeployedReplica;
3195     }
3196 
3197     Path getHdfsRegionDir() {
3198       if (hdfsEntry == null) {
3199         return null;
3200       }
3201       return hdfsEntry.hdfsRegionDir;
3202     }
3203 
3204     boolean containsOnlyHdfsEdits() {
3205       if (hdfsEntry == null) {
3206         return false;
3207       }
3208       return hdfsEntry.hdfsOnlyEdits;
3209     }
3210 
3211     boolean isHdfsRegioninfoPresent() {
3212       if (hdfsEntry == null) {
3213         return false;
3214       }
3215       return hdfsEntry.hdfsRegioninfoFilePresent;
3216     }
3217 
3218     long getModTime() {
3219       if (hdfsEntry == null) {
3220         return 0;
3221       }
3222       return hdfsEntry.hdfsRegionDirModTime;
3223     }
3224 
3225     HRegionInfo getHdfsHRI() {
3226       if (hdfsEntry == null) {
3227         return null;
3228       }
3229       return hdfsEntry.hri;
3230     }
3231 
3232     public void setSkipChecks(boolean skipChecks) {
3233       this.skipChecks = skipChecks;
3234     }
3235 
3236     public boolean isSkipChecks() {
3237       return skipChecks;
3238     }
3239 
3240     public void setMerged(boolean isMerged) {
3241       this.isMerged = isMerged;
3242     }
3243 
3244     public boolean isMerged() {
3245       return this.isMerged;
3246     }
3247   }
3248 
3249   final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3250     @Override
3251     public int compare(HbckInfo l, HbckInfo r) {
3252       if (l == r) {
3253         // same instance
3254         return 0;
3255       }
3256 
3257       int tableCompare = l.getTableName().compareTo(r.getTableName());
3258       if (tableCompare != 0) {
3259         return tableCompare;
3260       }
3261 
3262       int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3263           l.getStartKey(), r.getStartKey());
3264       if (startComparison != 0) {
3265         return startComparison;
3266       }
3267 
3268       // Special case for absolute endkey
3269       byte[] endKey = r.getEndKey();
3270       endKey = (endKey.length == 0) ? null : endKey;
3271       byte[] endKey2 = l.getEndKey();
3272       endKey2 = (endKey2.length == 0) ? null : endKey2;
3273       int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3274           endKey2,  endKey);
3275 
3276       if (endComparison != 0) {
3277         return endComparison;
3278       }
3279 
3280       // use regionId as tiebreaker.
3281       // Null is considered after all possible values so make it bigger.
3282       if (l.hdfsEntry == null && r.hdfsEntry == null) {
3283         return 0;
3284       }
3285       if (l.hdfsEntry == null && r.hdfsEntry != null) {
3286         return 1;
3287       }
3288       // l.hdfsEntry must not be null
3289       if (r.hdfsEntry == null) {
3290         return -1;
3291       }
3292       // both l.hdfsEntry and r.hdfsEntry must not be null.
3293       return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3294     }
3295   };
3296 
3297   /**
3298    * Prints summary of all tables found on the system.
3299    */
3300   private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3301     StringBuilder sb = new StringBuilder();
3302     errors.print("Summary:");
3303     for (TableInfo tInfo : tablesInfo.values()) {
3304       if (errors.tableHasErrors(tInfo)) {
3305         errors.print("Table " + tInfo.getName() + " is inconsistent.");
3306       } else {
3307         errors.print("  " + tInfo.getName() + " is okay.");
3308       }
3309       errors.print("    Number of regions: " + tInfo.getNumRegions());
3310       sb.setLength(0); // clear out existing buffer, if any.
3311       sb.append("    Deployed on: ");
3312       for (ServerName server : tInfo.deployedOn) {
3313         sb.append(" " + server.toString());
3314       }
3315       errors.print(sb.toString());
3316     }
3317   }
3318 
3319   static ErrorReporter getErrorReporter(
3320       final Configuration conf) throws ClassNotFoundException {
3321     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3322     return (ErrorReporter)ReflectionUtils.newInstance(reporter, conf);
3323   }
3324 
3325   public interface ErrorReporter {
3326     enum ERROR_CODE {
3327       UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3328       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
3329       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3330       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3331       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3332       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3333       WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR
3334     }
3335     void clear();
3336     void report(String message);
3337     void reportError(String message);
3338     void reportError(ERROR_CODE errorCode, String message);
3339     void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3340     void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3341     void reportError(
3342       ERROR_CODE errorCode,
3343       String message,
3344       TableInfo table,
3345       HbckInfo info1,
3346       HbckInfo info2
3347     );
3348     int summarize();
3349     void detail(String details);
3350     ArrayList<ERROR_CODE> getErrorList();
3351     void progress();
3352     void print(String message);
3353     void resetErrors();
3354     boolean tableHasErrors(TableInfo table);
3355   }
3356 
3357   static class PrintingErrorReporter implements ErrorReporter {
3358     public int errorCount = 0;
3359     private int showProgress;
3360 
3361     Set<TableInfo> errorTables = new HashSet<TableInfo>();
3362 
3363     // for use by unit tests to verify which errors were discovered
3364     private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3365 
3366     @Override
3367     public void clear() {
3368       errorTables.clear();
3369       errorList.clear();
3370       errorCount = 0;
3371     }
3372 
3373     @Override
3374     public synchronized void reportError(ERROR_CODE errorCode, String message) {
3375       if (errorCode == ERROR_CODE.WRONG_USAGE) {
3376         System.err.println(message);
3377         return;
3378       }
3379 
3380       errorList.add(errorCode);
3381       if (!summary) {
3382         System.out.println("ERROR: " + message);
3383       }
3384       errorCount++;
3385       showProgress = 0;
3386     }
3387 
3388     @Override
3389     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
3390       errorTables.add(table);
3391       reportError(errorCode, message);
3392     }
3393 
3394     @Override
3395     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3396                                          HbckInfo info) {
3397       errorTables.add(table);
3398       String reference = "(region " + info.getRegionNameAsString() + ")";
3399       reportError(errorCode, reference + " " + message);
3400     }
3401 
3402     @Override
3403     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3404                                          HbckInfo info1, HbckInfo info2) {
3405       errorTables.add(table);
3406       String reference = "(regions " + info1.getRegionNameAsString()
3407           + " and " + info2.getRegionNameAsString() + ")";
3408       reportError(errorCode, reference + " " + message);
3409     }
3410 
3411     @Override
3412     public synchronized void reportError(String message) {
3413       reportError(ERROR_CODE.UNKNOWN, message);
3414     }
3415 
3416     /**
3417      * Report error information, but do not increment the error count.  Intended for cases
3418      * where the actual error would have been reported previously.
3419      * @param message
3420      */
3421     @Override
3422     public synchronized void report(String message) {
3423       if (! summary) {
3424         System.out.println("ERROR: " + message);
3425       }
3426       showProgress = 0;
3427     }
3428 
3429     @Override
3430     public synchronized int summarize() {
3431       System.out.println(Integer.toString(errorCount) +
3432                          " inconsistencies detected.");
3433       if (errorCount == 0) {
3434         System.out.println("Status: OK");
3435         return 0;
3436       } else {
3437         System.out.println("Status: INCONSISTENT");
3438         return -1;
3439       }
3440     }
3441 
3442     @Override
3443     public ArrayList<ERROR_CODE> getErrorList() {
3444       return errorList;
3445     }
3446 
3447     @Override
3448     public synchronized void print(String message) {
3449       if (!summary) {
3450         System.out.println(message);
3451       }
3452     }
3453 
3454     @Override
3455     public boolean tableHasErrors(TableInfo table) {
3456       return errorTables.contains(table);
3457     }
3458 
3459     @Override
3460     public void resetErrors() {
3461       errorCount = 0;
3462     }
3463 
3464     @Override
3465     public synchronized void detail(String message) {
3466       if (details) {
3467         System.out.println(message);
3468       }
3469       showProgress = 0;
3470     }
3471 
3472     @Override
3473     public synchronized void progress() {
3474       if (showProgress++ == 10) {
3475         if (!summary) {
3476           System.out.print(".");
3477         }
3478         showProgress = 0;
3479       }
3480     }
3481   }
3482 
3483   /**
3484    * Contact a region server and get all information from it
3485    */
3486   static class WorkItemRegion implements Callable<Void> {
3487     private HBaseFsck hbck;
3488     private ServerName rsinfo;
3489     private ErrorReporter errors;
3490     private HConnection connection;
3491 
3492     WorkItemRegion(HBaseFsck hbck, ServerName info,
3493                    ErrorReporter errors, HConnection connection) {
3494       this.hbck = hbck;
3495       this.rsinfo = info;
3496       this.errors = errors;
3497       this.connection = connection;
3498     }
3499 
3500     @Override
3501     public synchronized Void call() throws IOException {
3502       errors.progress();
3503       try {
3504         BlockingInterface server = connection.getAdmin(rsinfo);
3505 
3506         // list all online regions from this region server
3507         List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
3508         regions = filterRegions(regions);
3509 
3510         if (details) {
3511           errors.detail("RegionServer: " + rsinfo.getServerName() +
3512                            " number of regions: " + regions.size());
3513           for (HRegionInfo rinfo: regions) {
3514             errors.detail("  " + rinfo.getRegionNameAsString() +
3515                              " id: " + rinfo.getRegionId() +
3516                              " encoded_name: " + rinfo.getEncodedName() +
3517                              " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
3518                              " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
3519           }
3520         }
3521 
3522         // check to see if the existence of this region matches the region in META
3523         for (HRegionInfo r:regions) {
3524           HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
3525           hbi.addServer(r, rsinfo);
3526         }
3527       } catch (IOException e) {          // unable to connect to the region server.
3528         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
3529           " Unable to fetch region information. " + e);
3530         throw e;
3531       }
3532       return null;
3533     }
3534 
3535     private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
3536       List<HRegionInfo> ret = Lists.newArrayList();
3537       for (HRegionInfo hri : regions) {
3538         if (hri.isMetaTable() || (!hbck.checkMetaOnly
3539             && hbck.isTableIncluded(hri.getTable()))) {
3540           ret.add(hri);
3541         }
3542       }
3543       return ret;
3544     }
3545   }
3546 
3547   /**
3548    * Contact hdfs and get all information about specified table directory into
3549    * regioninfo list.
3550    */
3551   static class WorkItemHdfsDir implements Callable<Void> {
3552     private HBaseFsck hbck;
3553     private FileStatus tableDir;
3554     private ErrorReporter errors;
3555     private FileSystem fs;
3556 
3557     WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
3558                     FileStatus status) {
3559       this.hbck = hbck;
3560       this.fs = fs;
3561       this.tableDir = status;
3562       this.errors = errors;
3563     }
3564 
3565     @Override
3566     public synchronized Void call() throws IOException {
3567       try {
3568         // level 2: <HBASE_DIR>/<table>/*
3569         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
3570         for (FileStatus regionDir : regionDirs) {
3571           String encodedName = regionDir.getPath().getName();
3572           // ignore directories that aren't hexadecimal
3573           if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
3574             continue;
3575           }
3576 
3577           LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
3578           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
3579           HdfsEntry he = new HdfsEntry();
3580           synchronized (hbi) {
3581             if (hbi.getHdfsRegionDir() != null) {
3582               errors.print("Directory " + encodedName + " duplicate??" +
3583                            hbi.getHdfsRegionDir());
3584             }
3585 
3586             he.hdfsRegionDir = regionDir.getPath();
3587             he.hdfsRegionDirModTime = regionDir.getModificationTime();
3588             Path regioninfoFile = new Path(he.hdfsRegionDir, HRegionFileSystem.REGION_INFO_FILE);
3589             he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
3590             // we add to orphan list when we attempt to read .regioninfo
3591 
3592             // Set a flag if this region contains only edits
3593             // This is special case if a region is left after split
3594             he.hdfsOnlyEdits = true;
3595             FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
3596             Path ePath = HLogUtil.getRegionDirRecoveredEditsDir(regionDir.getPath());
3597             for (FileStatus subDir : subDirs) {
3598               String sdName = subDir.getPath().getName();
3599               if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
3600                 he.hdfsOnlyEdits = false;
3601                 break;
3602               }
3603             }
3604             hbi.hdfsEntry = he;
3605           }
3606         }
3607       } catch (IOException e) {
3608         // unable to connect to the region server.
3609         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
3610             + tableDir.getPath().getName()
3611             + " Unable to fetch region information. " + e);
3612         throw e;
3613       }
3614       return null;
3615     }
3616   }
3617 
3618   /**
3619    * Contact hdfs and get all information about specified table directory into
3620    * regioninfo list.
3621    */
3622   static class WorkItemHdfsRegionInfo implements Callable<Void> {
3623     private HbckInfo hbi;
3624     private HBaseFsck hbck;
3625     private ErrorReporter errors;
3626 
3627     WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
3628       this.hbi = hbi;
3629       this.hbck = hbck;
3630       this.errors = errors;
3631     }
3632 
3633     @Override
3634     public synchronized Void call() throws IOException {
3635       // only load entries that haven't been loaded yet.
3636       if (hbi.getHdfsHRI() == null) {
3637         try {
3638           hbck.loadHdfsRegioninfo(hbi);
3639         } catch (IOException ioe) {
3640           String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
3641               + hbi.getTableName() + " in hdfs dir "
3642               + hbi.getHdfsRegionDir()
3643               + "!  It may be an invalid format or version file.  Treating as "
3644               + "an orphaned regiondir.";
3645           errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
3646           try {
3647             hbck.debugLsr(hbi.getHdfsRegionDir());
3648           } catch (IOException ioe2) {
3649             LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
3650             throw ioe2;
3651           }
3652           hbck.orphanHdfsDirs.add(hbi);
3653           throw ioe;
3654         }
3655       }
3656       return null;
3657     }
3658   };
3659 
3660   /**
3661    * Display the full report from fsck. This displays all live and dead region
3662    * servers, and all known regions.
3663    */
3664   public static void setDisplayFullReport() {
3665     details = true;
3666   }
3667 
3668   /**
3669    * Set summary mode.
3670    * Print only summary of the tables and status (OK or INCONSISTENT)
3671    */
3672   void setSummary() {
3673     summary = true;
3674   }
3675 
3676   /**
3677    * Set hbase:meta check mode.
3678    * Print only info about hbase:meta table deployment/state
3679    */
3680   void setCheckMetaOnly() {
3681     checkMetaOnly = true;
3682   }
3683 
3684   /**
3685    * Set region boundaries check mode.
3686    */
3687   void setRegionBoundariesCheck() {
3688     checkRegionBoundaries = true;
3689   }
3690 
3691   /**
3692    * Set table locks fix mode.
3693    * Delete table locks held for a long time
3694    */
3695   public void setFixTableLocks(boolean shouldFix) {
3696     fixTableLocks = shouldFix;
3697   }
3698 
3699   /**
3700    * Check if we should rerun fsck again. This checks if we've tried to
3701    * fix something and we should rerun fsck tool again.
3702    * Display the full report from fsck. This displays all live and dead
3703    * region servers, and all known regions.
3704    */
3705   void setShouldRerun() {
3706     rerun = true;
3707   }
3708 
3709   boolean shouldRerun() {
3710     return rerun;
3711   }
3712 
3713   /**
3714    * Fix inconsistencies found by fsck. This should try to fix errors (if any)
3715    * found by fsck utility.
3716    */
3717   public void setFixAssignments(boolean shouldFix) {
3718     fixAssignments = shouldFix;
3719   }
3720 
3721   boolean shouldFixAssignments() {
3722     return fixAssignments;
3723   }
3724 
3725   public void setFixMeta(boolean shouldFix) {
3726     fixMeta = shouldFix;
3727   }
3728 
3729   boolean shouldFixMeta() {
3730     return fixMeta;
3731   }
3732 
3733   public void setFixEmptyMetaCells(boolean shouldFix) {
3734     fixEmptyMetaCells = shouldFix;
3735   }
3736 
3737   boolean shouldFixEmptyMetaCells() {
3738     return fixEmptyMetaCells;
3739   }
3740 
3741   public void setCheckHdfs(boolean checking) {
3742     checkHdfs = checking;
3743   }
3744 
3745   boolean shouldCheckHdfs() {
3746     return checkHdfs;
3747   }
3748 
3749   public void setFixHdfsHoles(boolean shouldFix) {
3750     fixHdfsHoles = shouldFix;
3751   }
3752 
3753   boolean shouldFixHdfsHoles() {
3754     return fixHdfsHoles;
3755   }
3756 
3757   public void setFixTableOrphans(boolean shouldFix) {
3758     fixTableOrphans = shouldFix;
3759   }
3760 
3761   boolean shouldFixTableOrphans() {
3762     return fixTableOrphans;
3763   }
3764 
3765   public void setFixHdfsOverlaps(boolean shouldFix) {
3766     fixHdfsOverlaps = shouldFix;
3767   }
3768 
3769   boolean shouldFixHdfsOverlaps() {
3770     return fixHdfsOverlaps;
3771   }
3772 
3773   public void setFixHdfsOrphans(boolean shouldFix) {
3774     fixHdfsOrphans = shouldFix;
3775   }
3776 
3777   boolean shouldFixHdfsOrphans() {
3778     return fixHdfsOrphans;
3779   }
3780 
3781   public void setFixVersionFile(boolean shouldFix) {
3782     fixVersionFile = shouldFix;
3783   }
3784 
3785   public boolean shouldFixVersionFile() {
3786     return fixVersionFile;
3787   }
3788 
3789   public void setSidelineBigOverlaps(boolean sbo) {
3790     this.sidelineBigOverlaps = sbo;
3791   }
3792 
3793   public boolean shouldSidelineBigOverlaps() {
3794     return sidelineBigOverlaps;
3795   }
3796 
3797   public void setFixSplitParents(boolean shouldFix) {
3798     fixSplitParents = shouldFix;
3799   }
3800 
3801   boolean shouldFixSplitParents() {
3802     return fixSplitParents;
3803   }
3804 
3805   public void setFixReferenceFiles(boolean shouldFix) {
3806     fixReferenceFiles = shouldFix;
3807   }
3808 
3809   boolean shouldFixReferenceFiles() {
3810     return fixReferenceFiles;
3811   }
3812 
3813   public boolean shouldIgnorePreCheckPermission() {
3814     return ignorePreCheckPermission;
3815   }
3816 
3817   public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
3818     this.ignorePreCheckPermission = ignorePreCheckPermission;
3819   }
3820 
3821   /**
3822    * @param mm maximum number of regions to merge into a single region.
3823    */
3824   public void setMaxMerge(int mm) {
3825     this.maxMerge = mm;
3826   }
3827 
3828   public int getMaxMerge() {
3829     return maxMerge;
3830   }
3831 
3832   public void setMaxOverlapsToSideline(int mo) {
3833     this.maxOverlapsToSideline = mo;
3834   }
3835 
3836   public int getMaxOverlapsToSideline() {
3837     return maxOverlapsToSideline;
3838   }
3839 
3840   /**
3841    * Only check/fix tables specified by the list,
3842    * Empty list means all tables are included.
3843    */
3844   boolean isTableIncluded(TableName table) {
3845     return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
3846   }
3847 
3848   public void includeTable(TableName table) {
3849     tablesIncluded.add(table);
3850   }
3851 
3852   Set<TableName> getIncludedTables() {
3853     return new HashSet<TableName>(tablesIncluded);
3854   }
3855 
3856   /**
3857    * We are interested in only those tables that have not changed their state in
3858    * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
3859    * @param seconds - the time in seconds
3860    */
3861   public void setTimeLag(long seconds) {
3862     timelag = seconds * 1000; // convert to milliseconds
3863   }
3864 
3865   /**
3866    *
3867    * @param sidelineDir - HDFS path to sideline data
3868    */
3869   public void setSidelineDir(String sidelineDir) {
3870     this.sidelineDir = new Path(sidelineDir);
3871   }
3872 
3873   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
3874     return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
3875   }
3876 
3877   public HFileCorruptionChecker getHFilecorruptionChecker() {
3878     return hfcc;
3879   }
3880 
3881   public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
3882     this.hfcc = hfcc;
3883   }
3884 
3885   public void setRetCode(int code) {
3886     this.retcode = code;
3887   }
3888 
3889   public int getRetCode() {
3890     return retcode;
3891   }
3892 
3893   protected HBaseFsck printUsageAndExit() {
3894     StringWriter sw = new StringWriter(2048);
3895     PrintWriter out = new PrintWriter(sw);
3896     out.println("Usage: fsck [opts] {only tables}");
3897     out.println(" where [opts] are:");
3898     out.println("   -help Display help options (this)");
3899     out.println("   -details Display full report of all regions.");
3900     out.println("   -timelag <timeInSeconds>  Process only regions that " +
3901                        " have not experienced any metadata updates in the last " +
3902                        " <timeInSeconds> seconds.");
3903     out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
3904         " before checking if the fix worked if run with -fix");
3905     out.println("   -summary Print only summary of the tables and status.");
3906     out.println("   -metaonly Only check the state of the hbase:meta table.");
3907     out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
3908     out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
3909 
3910     out.println("");
3911     out.println("  Metadata Repair options: (expert features, use with caution!)");
3912     out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
3913     out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
3914     out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
3915     out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
3916         + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
3917     out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
3918     out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
3919     out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
3920     out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
3921     out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
3922     out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
3923     out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
3924     out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
3925     out.println("   -fixSplitParents  Try to force offline split parents to be online.");
3926     out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
3927     out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
3928     out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
3929         + " (empty REGIONINFO_QUALIFIER rows)");
3930 
3931     out.println("");
3932     out.println("  Datafile Repair options: (expert features, use with caution!)");
3933     out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
3934     out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
3935 
3936     out.println("");
3937     out.println("  Metadata Repair shortcuts");
3938     out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
3939         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles -fixTableLocks");
3940     out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
3941 
3942     out.println("");
3943     out.println("  Table lock options");
3944     out.println("   -fixTableLocks    Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
3945 
3946     out.flush();
3947     errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
3948 
3949     setRetCode(-2);
3950     return this;
3951   }
3952 
3953   /**
3954    * Main program
3955    *
3956    * @param args
3957    * @throws Exception
3958    */
3959   public static void main(String[] args) throws Exception {
3960     // create a fsck object
3961     Configuration conf = HBaseConfiguration.create();
3962     Path hbasedir = FSUtils.getRootDir(conf);
3963     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
3964     FSUtils.setFsDefault(conf, new Path(defaultFs));
3965 
3966     int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
3967     System.exit(ret);
3968   }
3969 
3970   /**
3971    * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
3972    */
3973   static class HBaseFsckTool extends Configured implements Tool {
3974     HBaseFsckTool(Configuration conf) { super(conf); }
3975     @Override
3976     public int run(String[] args) throws Exception {
3977       HBaseFsck hbck = new HBaseFsck(getConf());
3978       hbck.exec(hbck.executor, args);
3979       return hbck.getRetCode();
3980     }
3981   };
3982 
3983 
3984   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
3985     ServiceException, InterruptedException {
3986     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
3987 
3988     boolean checkCorruptHFiles = false;
3989     boolean sidelineCorruptHFiles = false;
3990 
3991     // Process command-line args.
3992     for (int i = 0; i < args.length; i++) {
3993       String cmd = args[i];
3994       if (cmd.equals("-help") || cmd.equals("-h")) {
3995         return printUsageAndExit();
3996       } else if (cmd.equals("-details")) {
3997         setDisplayFullReport();
3998       } else if (cmd.equals("-timelag")) {
3999         if (i == args.length - 1) {
4000           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
4001           return printUsageAndExit();
4002         }
4003         try {
4004           long timelag = Long.parseLong(args[i+1]);
4005           setTimeLag(timelag);
4006         } catch (NumberFormatException e) {
4007           errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
4008           return printUsageAndExit();
4009         }
4010         i++;
4011       } else if (cmd.equals("-sleepBeforeRerun")) {
4012         if (i == args.length - 1) {
4013           errors.reportError(ERROR_CODE.WRONG_USAGE,
4014             "HBaseFsck: -sleepBeforeRerun needs a value.");
4015           return printUsageAndExit();
4016         }
4017         try {
4018           sleepBeforeRerun = Long.parseLong(args[i+1]);
4019         } catch (NumberFormatException e) {
4020           errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
4021           return printUsageAndExit();
4022         }
4023         i++;
4024       } else if (cmd.equals("-sidelineDir")) {
4025         if (i == args.length - 1) {
4026           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
4027           return printUsageAndExit();
4028         }
4029         i++;
4030         setSidelineDir(args[i]);
4031       } else if (cmd.equals("-fix")) {
4032         errors.reportError(ERROR_CODE.WRONG_USAGE,
4033           "This option is deprecated, please use  -fixAssignments instead.");
4034         setFixAssignments(true);
4035       } else if (cmd.equals("-fixAssignments")) {
4036         setFixAssignments(true);
4037       } else if (cmd.equals("-fixMeta")) {
4038         setFixMeta(true);
4039       } else if (cmd.equals("-noHdfsChecking")) {
4040         setCheckHdfs(false);
4041       } else if (cmd.equals("-fixHdfsHoles")) {
4042         setFixHdfsHoles(true);
4043       } else if (cmd.equals("-fixHdfsOrphans")) {
4044         setFixHdfsOrphans(true);
4045       } else if (cmd.equals("-fixTableOrphans")) {
4046         setFixTableOrphans(true);
4047       } else if (cmd.equals("-fixHdfsOverlaps")) {
4048         setFixHdfsOverlaps(true);
4049       } else if (cmd.equals("-fixVersionFile")) {
4050         setFixVersionFile(true);
4051       } else if (cmd.equals("-sidelineBigOverlaps")) {
4052         setSidelineBigOverlaps(true);
4053       } else if (cmd.equals("-fixSplitParents")) {
4054         setFixSplitParents(true);
4055       } else if (cmd.equals("-ignorePreCheckPermission")) {
4056         setIgnorePreCheckPermission(true);
4057       } else if (cmd.equals("-checkCorruptHFiles")) {
4058         checkCorruptHFiles = true;
4059       } else if (cmd.equals("-sidelineCorruptHFiles")) {
4060         sidelineCorruptHFiles = true;
4061       } else if (cmd.equals("-fixReferenceFiles")) {
4062         setFixReferenceFiles(true);
4063       } else if (cmd.equals("-fixEmptyMetaCells")) {
4064         setFixEmptyMetaCells(true);
4065       } else if (cmd.equals("-repair")) {
4066         // this attempts to merge overlapping hdfs regions, needs testing
4067         // under load
4068         setFixHdfsHoles(true);
4069         setFixHdfsOrphans(true);
4070         setFixMeta(true);
4071         setFixAssignments(true);
4072         setFixHdfsOverlaps(true);
4073         setFixVersionFile(true);
4074         setSidelineBigOverlaps(true);
4075         setFixSplitParents(false);
4076         setCheckHdfs(true);
4077         setFixReferenceFiles(true);
4078         setFixTableLocks(true);
4079       } else if (cmd.equals("-repairHoles")) {
4080         // this will make all missing hdfs regions available but may lose data
4081         setFixHdfsHoles(true);
4082         setFixHdfsOrphans(false);
4083         setFixMeta(true);
4084         setFixAssignments(true);
4085         setFixHdfsOverlaps(false);
4086         setSidelineBigOverlaps(false);
4087         setFixSplitParents(false);
4088         setCheckHdfs(true);
4089       } else if (cmd.equals("-maxOverlapsToSideline")) {
4090         if (i == args.length - 1) {
4091           errors.reportError(ERROR_CODE.WRONG_USAGE,
4092             "-maxOverlapsToSideline needs a numeric value argument.");
4093           return printUsageAndExit();
4094         }
4095         try {
4096           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
4097           setMaxOverlapsToSideline(maxOverlapsToSideline);
4098         } catch (NumberFormatException e) {
4099           errors.reportError(ERROR_CODE.WRONG_USAGE,
4100             "-maxOverlapsToSideline needs a numeric value argument.");
4101           return printUsageAndExit();
4102         }
4103         i++;
4104       } else if (cmd.equals("-maxMerge")) {
4105         if (i == args.length - 1) {
4106           errors.reportError(ERROR_CODE.WRONG_USAGE,
4107             "-maxMerge needs a numeric value argument.");
4108           return printUsageAndExit();
4109         }
4110         try {
4111           int maxMerge = Integer.parseInt(args[i+1]);
4112           setMaxMerge(maxMerge);
4113         } catch (NumberFormatException e) {
4114           errors.reportError(ERROR_CODE.WRONG_USAGE,
4115             "-maxMerge needs a numeric value argument.");
4116           return printUsageAndExit();
4117         }
4118         i++;
4119       } else if (cmd.equals("-summary")) {
4120         setSummary();
4121       } else if (cmd.equals("-metaonly")) {
4122         setCheckMetaOnly();
4123       } else if (cmd.equals("-boundaries")) {
4124         setRegionBoundariesCheck();
4125       } else if (cmd.equals("-fixTableLocks")) {
4126         setFixTableLocks(true);
4127       } else if (cmd.startsWith("-")) {
4128         errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4129         return printUsageAndExit();
4130       } else {
4131         includeTable(TableName.valueOf(cmd));
4132         errors.print("Allow checking/fixes for table: " + cmd);
4133       }
4134     }
4135 
4136     // pre-check current user has FS write permission or not
4137     try {
4138       preCheckPermission();
4139     } catch (AccessDeniedException ace) {
4140       Runtime.getRuntime().exit(-1);
4141     } catch (IOException ioe) {
4142       Runtime.getRuntime().exit(-1);
4143     }
4144 
4145     // do the real work of hbck
4146     connect();
4147 
4148     try {
4149       // if corrupt file mode is on, first fix them since they may be opened later
4150       if (checkCorruptHFiles || sidelineCorruptHFiles) {
4151         LOG.info("Checking all hfiles for corruption");
4152         HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4153         setHFileCorruptionChecker(hfcc); // so we can get result
4154         Collection<TableName> tables = getIncludedTables();
4155         Collection<Path> tableDirs = new ArrayList<Path>();
4156         Path rootdir = FSUtils.getRootDir(getConf());
4157         if (tables.size() > 0) {
4158           for (TableName t : tables) {
4159             tableDirs.add(FSUtils.getTableDir(rootdir, t));
4160           }
4161         } else {
4162           tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4163         }
4164         hfcc.checkTables(tableDirs);
4165         hfcc.report(errors);
4166       }
4167 
4168       // check and fix table integrity, region consistency.
4169       int code = onlineHbck();
4170       setRetCode(code);
4171       // If we have changed the HBase state it is better to run hbck again
4172       // to see if we haven't broken something else in the process.
4173       // We run it only once more because otherwise we can easily fall into
4174       // an infinite loop.
4175       if (shouldRerun()) {
4176         try {
4177           LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4178           Thread.sleep(sleepBeforeRerun);
4179         } catch (InterruptedException ie) {
4180           LOG.warn("Interrupted while sleeping");
4181           return this;
4182         }
4183         // Just report
4184         setFixAssignments(false);
4185         setFixMeta(false);
4186         setFixHdfsHoles(false);
4187         setFixHdfsOverlaps(false);
4188         setFixVersionFile(false);
4189         setFixTableOrphans(false);
4190         errors.resetErrors();
4191         code = onlineHbck();
4192         setRetCode(code);
4193       }
4194     } finally {
4195       IOUtils.cleanup(null, connection, meta, admin);
4196     }
4197     return this;
4198   }
4199 
4200   /**
4201    * ls -r for debugging purposes
4202    */
4203   void debugLsr(Path p) throws IOException {
4204     debugLsr(getConf(), p, errors);
4205   }
4206 
4207   /**
4208    * ls -r for debugging purposes
4209    */
4210   public static void debugLsr(Configuration conf,
4211       Path p) throws IOException {
4212     debugLsr(conf, p, new PrintingErrorReporter());
4213   }
4214 
4215   /**
4216    * ls -r for debugging purposes
4217    */
4218   public static void debugLsr(Configuration conf,
4219       Path p, ErrorReporter errors) throws IOException {
4220     if (!LOG.isDebugEnabled() || p == null) {
4221       return;
4222     }
4223     FileSystem fs = p.getFileSystem(conf);
4224 
4225     if (!fs.exists(p)) {
4226       // nothing
4227       return;
4228     }
4229     errors.print(p.toString());
4230 
4231     if (fs.isFile(p)) {
4232       return;
4233     }
4234 
4235     if (fs.getFileStatus(p).isDirectory()) {
4236       FileStatus[] fss= fs.listStatus(p);
4237       for (FileStatus status : fss) {
4238         debugLsr(conf, status.getPath(), errors);
4239       }
4240     }
4241   }
4242 }