View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import java.io.FileNotFoundException;
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.io.PrintWriter;
24  import java.io.StringWriter;
25  import java.net.URI;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.Collection;
29  import java.util.Collections;
30  import java.util.Comparator;
31  import java.util.HashMap;
32  import java.util.HashSet;
33  import java.util.Iterator;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.Map.Entry;
37  import java.util.Set;
38  import java.util.SortedMap;
39  import java.util.SortedSet;
40  import java.util.TreeMap;
41  import java.util.TreeSet;
42  import java.util.concurrent.Callable;
43  import java.util.concurrent.ConcurrentSkipListMap;
44  import java.util.concurrent.ExecutionException;
45  import java.util.concurrent.ExecutorService;
46  import java.util.concurrent.Future;
47  import java.util.concurrent.ScheduledThreadPoolExecutor;
48  import java.util.concurrent.atomic.AtomicInteger;
49  
50  import org.apache.commons.logging.Log;
51  import org.apache.commons.logging.LogFactory;
52  import org.apache.hadoop.classification.InterfaceAudience;
53  import org.apache.hadoop.classification.InterfaceStability;
54  import org.apache.hadoop.conf.Configuration;
55  import org.apache.hadoop.conf.Configured;
56  import org.apache.hadoop.fs.FileStatus;
57  import org.apache.hadoop.fs.FileSystem;
58  import org.apache.hadoop.fs.Path;
59  import org.apache.hadoop.fs.permission.FsAction;
60  import org.apache.hadoop.hbase.Abortable;
61  import org.apache.hadoop.hbase.Cell;
62  import org.apache.hadoop.hbase.ClusterStatus;
63  import org.apache.hadoop.hbase.HBaseConfiguration;
64  import org.apache.hadoop.hbase.HColumnDescriptor;
65  import org.apache.hadoop.hbase.HConstants;
66  import org.apache.hadoop.hbase.HRegionInfo;
67  import org.apache.hadoop.hbase.HRegionLocation;
68  import org.apache.hadoop.hbase.HTableDescriptor;
69  import org.apache.hadoop.hbase.KeyValue;
70  import org.apache.hadoop.hbase.MasterNotRunningException;
71  import org.apache.hadoop.hbase.ServerName;
72  import org.apache.hadoop.hbase.TableName;
73  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
74  import org.apache.hadoop.hbase.MetaTableAccessor;
75  import org.apache.hadoop.hbase.client.Delete;
76  import org.apache.hadoop.hbase.client.Get;
77  import org.apache.hadoop.hbase.client.HBaseAdmin;
78  import org.apache.hadoop.hbase.client.HConnectable;
79  import org.apache.hadoop.hbase.client.HConnection;
80  import org.apache.hadoop.hbase.client.HConnectionManager;
81  import org.apache.hadoop.hbase.client.HTable;
82  import org.apache.hadoop.hbase.client.MetaScanner;
83  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
84  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
85  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
86  import org.apache.hadoop.hbase.client.Put;
87  import org.apache.hadoop.hbase.client.Result;
88  import org.apache.hadoop.hbase.client.RowMutations;
89  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
90  import org.apache.hadoop.hbase.io.hfile.HFile;
91  import org.apache.hadoop.hbase.master.MasterFileSystem;
92  import org.apache.hadoop.hbase.master.RegionState;
93  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
94  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
95  import org.apache.hadoop.hbase.regionserver.HRegion;
96  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
97  import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
98  import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
99  import org.apache.hadoop.hbase.security.UserProvider;
100 import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
101 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
102 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
103 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;
104 import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
105 import org.apache.hadoop.hbase.util.hbck.TableLockChecker;
106 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
107 import org.apache.hadoop.hbase.zookeeper.ZKTableStateClientSideReader;
108 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
109 import org.apache.hadoop.io.IOUtils;
110 import org.apache.hadoop.security.AccessControlException;
111 import org.apache.hadoop.security.UserGroupInformation;
112 import org.apache.hadoop.util.ReflectionUtils;
113 import org.apache.hadoop.util.Tool;
114 import org.apache.hadoop.util.ToolRunner;
115 import org.apache.zookeeper.KeeperException;
116 
117 import com.google.common.base.Joiner;
118 import com.google.common.base.Preconditions;
119 import com.google.common.collect.Lists;
120 import com.google.common.collect.Multimap;
121 import com.google.common.collect.TreeMultimap;
122 import com.google.protobuf.ServiceException;
123 
124 /**
125  * HBaseFsck (hbck) is a tool for checking and repairing region consistency and
126  * table integrity problems in a corrupted HBase.
127  * <p>
128  * Region consistency checks verify that hbase:meta, region deployment on region
129  * servers and the state of data in HDFS (.regioninfo files) all are in
130  * accordance.
131  * <p>
132  * Table integrity checks verify that all possible row keys resolve to exactly
133  * one region of a table.  This means there are no individual degenerate
134  * or backwards regions; no holes between regions; and that there are no
135  * overlapping regions.
136  * <p>
137  * The general repair strategy works in two phases:
138  * <ol>
139  * <li> Repair Table Integrity on HDFS. (merge or fabricate regions)
140  * <li> Repair Region Consistency with hbase:meta and assignments
141  * </ol>
142  * <p>
143  * For table integrity repairs, the tables' region directories are scanned
144  * for .regioninfo files.  Each table's integrity is then verified.  If there
145  * are any orphan regions (regions with no .regioninfo files) or holes, new
146  * regions are fabricated.  Backwards regions are sidelined as well as empty
147  * degenerate (endkey==startkey) regions.  If there are any overlapping regions,
148  * a new region is created and all data is merged into the new region.
149  * <p>
150  * Table integrity repairs deal solely with HDFS and could potentially be done
151  * offline -- the hbase region servers or master do not need to be running.
152  * This phase can eventually be used to completely reconstruct the hbase:meta table in
153  * an offline fashion.
154  * <p>
155  * Region consistency requires three conditions -- 1) valid .regioninfo file
156  * present in an HDFS region dir,  2) valid row with .regioninfo data in META,
157  * and 3) a region is deployed only at the regionserver that was assigned to
158  * with proper state in the master.
159  * <p>
160  * Region consistency repairs require hbase to be online so that hbck can
161  * contact the HBase master and region servers.  The hbck#connect() method must
162  * first be called successfully.  Much of the region consistency information
163  * is transient and less risky to repair.
164  * <p>
165  * If hbck is run from the command line, there are a handful of arguments that
166  * can be used to limit the kinds of repairs hbck will do.  See the code in
167  * {@link #printUsageAndExit()} for more details.
168  */
169 @InterfaceAudience.Public
170 @InterfaceStability.Evolving
171 public class HBaseFsck extends Configured {
172   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
173   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
174   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
175   private static boolean rsSupportsOffline = true;
176   private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
177   private static final int DEFAULT_MAX_MERGE = 5;
178   private static final String TO_BE_LOADED = "to_be_loaded";
179 
180   /**********************
181    * Internal resources
182    **********************/
183   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
184   private ClusterStatus status;
185   private HConnection connection;
186   private HBaseAdmin admin;
187   private HTable meta;
188   // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions
189   protected ExecutorService executor;
190   private long startMillis = System.currentTimeMillis();
191   private HFileCorruptionChecker hfcc;
192   private int retcode = 0;
193 
194   /***********
195    * Options
196    ***********/
197   private static boolean details = false; // do we display the full report
198   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
199   private boolean fixAssignments = false; // fix assignment errors?
200   private boolean fixMeta = false; // fix meta errors?
201   private boolean checkHdfs = true; // load and check fs consistency?
202   private boolean fixHdfsHoles = false; // fix fs holes?
203   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
204   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
205   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
206   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
207   private boolean fixSplitParents = false; // fix lingering split parents
208   private boolean fixReferenceFiles = false; // fix lingering reference store file
209   private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
210   private boolean fixTableLocks = false; // fix table locks which are expired
211 
212   // limit checking/fixes to listed tables, if empty attempt to check/fix all
213   // hbase:meta are always checked
214   private Set<TableName> tablesIncluded = new HashSet<TableName>();
215   private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
216   private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
217   private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
218   private Path sidelineDir = null;
219 
220   private boolean rerun = false; // if we tried to fix something, rerun hbck
221   private static boolean summary = false; // if we want to print less output
222   private boolean checkMetaOnly = false;
223   private boolean checkRegionBoundaries = false;
224   private boolean ignorePreCheckPermission = false; // if pre-check permission
225 
226   /*********
227    * State
228    *********/
229   final private ErrorReporter errors;
230   int fixes = 0;
231 
232   /**
233    * This map contains the state of all hbck items.  It maps from encoded region
234    * name to HbckInfo structure.  The information contained in HbckInfo is used
235    * to detect and correct consistency (hdfs/meta/deployment) problems.
236    */
237   private TreeMap<String, HbckInfo> regionInfoMap = new TreeMap<String, HbckInfo>();
238   private TreeSet<TableName> disabledTables =
239     new TreeSet<TableName>();
240   // Empty regioninfo qualifiers in hbase:meta
241   private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>();
242 
243   /**
244    * This map from Tablename -> TableInfo contains the structures necessary to
245    * detect table consistency problems (holes, dupes, overlaps).  It is sorted
246    * to prevent dupes.
247    *
248    * If tablesIncluded is empty, this map contains all tables.
249    * Otherwise, it contains only meta tables and tables in tablesIncluded,
250    * unless checkMetaOnly is specified, in which case, it contains only
251    * the meta table
252    */
253   private SortedMap<TableName, TableInfo> tablesInfo =
254       new ConcurrentSkipListMap<TableName, TableInfo>();
255 
256   /**
257    * When initially looking at HDFS, we attempt to find any orphaned data.
258    */
259   private List<HbckInfo> orphanHdfsDirs = Collections.synchronizedList(new ArrayList<HbckInfo>());
260 
261   private Map<TableName, Set<String>> orphanTableDirs =
262       new HashMap<TableName, Set<String>>();
263 
264   /**
265    * Constructor
266    *
267    * @param conf Configuration object
268    * @throws MasterNotRunningException if the master is not running
269    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
270    */
271   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
272       ZooKeeperConnectionException, IOException, ClassNotFoundException {
273     super(conf);
274     // make a copy, just to be sure we're not overriding someone else's config
275     setConf(HBaseConfiguration.create(getConf()));
276     // disable blockcache for tool invocation, see HBASE-10500
277     getConf().setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0);
278     errors = getErrorReporter(conf);
279 
280     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
281     executor = new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck"));
282   }
283 
284   /**
285    * Constructor
286    *
287    * @param conf
288    *          Configuration object
289    * @throws MasterNotRunningException
290    *           if the master is not running
291    * @throws ZooKeeperConnectionException
292    *           if unable to connect to ZooKeeper
293    */
294   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
295       ZooKeeperConnectionException, IOException, ClassNotFoundException {
296     super(conf);
297     errors = getErrorReporter(getConf());
298     this.executor = exec;
299   }
300 
301   /**
302    * To repair region consistency, one must call connect() in order to repair
303    * online state.
304    */
305   public void connect() throws IOException {
306     connection = HConnectionManager.createConnection(getConf());
307     admin = new HBaseAdmin(connection);
308     meta = new HTable(TableName.META_TABLE_NAME, connection);
309     status = admin.getClusterStatus();
310   }
311 
312   /**
313    * Get deployed regions according to the region servers.
314    */
315   private void loadDeployedRegions() throws IOException, InterruptedException {
316     // From the master, get a list of all known live region servers
317     Collection<ServerName> regionServers = status.getServers();
318     errors.print("Number of live region servers: " + regionServers.size());
319     if (details) {
320       for (ServerName rsinfo: regionServers) {
321         errors.print("  " + rsinfo.getServerName());
322       }
323     }
324 
325     // From the master, get a list of all dead region servers
326     Collection<ServerName> deadRegionServers = status.getDeadServerNames();
327     errors.print("Number of dead region servers: " + deadRegionServers.size());
328     if (details) {
329       for (ServerName name: deadRegionServers) {
330         errors.print("  " + name);
331       }
332     }
333 
334     // Print the current master name and state
335     errors.print("Master: " + status.getMaster());
336 
337     // Print the list of all backup masters
338     Collection<ServerName> backupMasters = status.getBackupMasters();
339     errors.print("Number of backup masters: " + backupMasters.size());
340     if (details) {
341       for (ServerName name: backupMasters) {
342         errors.print("  " + name);
343       }
344     }
345 
346     errors.print("Average load: " + status.getAverageLoad());
347     errors.print("Number of requests: " + status.getRequestsCount());
348     errors.print("Number of regions: " + status.getRegionsCount());
349 
350     Map<String, RegionState> rits = status.getRegionsInTransition();
351     errors.print("Number of regions in transition: " + rits.size());
352     if (details) {
353       for (RegionState state: rits.values()) {
354         errors.print("  " + state.toDescriptiveString());
355       }
356     }
357 
358     // Determine what's deployed
359     processRegionServers(regionServers);
360   }
361 
362   /**
363    * Clear the current state of hbck.
364    */
365   private void clearState() {
366     // Make sure regionInfo is empty before starting
367     fixes = 0;
368     regionInfoMap.clear();
369     emptyRegionInfoQualifiers.clear();
370     disabledTables.clear();
371     errors.clear();
372     tablesInfo.clear();
373     orphanHdfsDirs.clear();
374   }
375 
376   /**
377    * This repair method analyzes hbase data in hdfs and repairs it to satisfy
378    * the table integrity rules.  HBase doesn't need to be online for this
379    * operation to work.
380    */
381   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
382     // Initial pass to fix orphans.
383     if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
384         || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
385       LOG.info("Loading regioninfos HDFS");
386       // if nothing is happening this should always complete in two iterations.
387       int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
388       int curIter = 0;
389       do {
390         clearState(); // clears hbck state and reset fixes to 0 and.
391         // repair what's on HDFS
392         restoreHdfsIntegrity();
393         curIter++;// limit the number of iterations.
394       } while (fixes > 0 && curIter <= maxIterations);
395 
396       // Repairs should be done in the first iteration and verification in the second.
397       // If there are more than 2 passes, something funny has happened.
398       if (curIter > 2) {
399         if (curIter == maxIterations) {
400           LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "
401               + "Tables integrity may not be fully repaired!");
402         } else {
403           LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");
404         }
405       }
406     }
407   }
408 
409   /**
410    * This repair method requires the cluster to be online since it contacts
411    * region servers and the masters.  It makes each region's state in HDFS, in
412    * hbase:meta, and deployments consistent.
413    *
414    * @return If > 0 , number of errors detected, if < 0 there was an unrecoverable
415    * error.  If 0, we have a clean hbase.
416    */
417   public int onlineConsistencyRepair() throws IOException, KeeperException,
418     InterruptedException {
419     clearState();
420 
421     // get regions according to what is online on each RegionServer
422     loadDeployedRegions();
423     // check whether hbase:meta is deployed and online
424     recordMetaRegion();
425     // Check if hbase:meta is found only once and in the right place
426     if (!checkMetaRegion()) {
427       String errorMsg = "hbase:meta table is not consistent. ";
428       if (shouldFixAssignments()) {
429         errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";
430       } else {
431         errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";
432       }
433       errors.reportError(errorMsg + " Exiting...");
434       return -2;
435     }
436     // Not going with further consistency check for tables when hbase:meta itself is not consistent.
437     LOG.info("Loading regionsinfo from the hbase:meta table");
438     boolean success = loadMetaEntries();
439     if (!success) return -1;
440 
441     // Empty cells in hbase:meta?
442     reportEmptyMetaCells();
443 
444     // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta
445     if (shouldFixEmptyMetaCells()) {
446       fixEmptyMetaCells();
447     }
448 
449     // get a list of all tables that have not changed recently.
450     if (!checkMetaOnly) {
451       reportTablesInFlux();
452     }
453 
454     // load regiondirs and regioninfos from HDFS
455     if (shouldCheckHdfs()) {
456       loadHdfsRegionDirs();
457       loadHdfsRegionInfos();
458     }
459 
460     // Get disabled tables from ZooKeeper
461     loadDisabledTables();
462 
463     // fix the orphan tables
464     fixOrphanTables();
465 
466     // Check and fix consistency
467     checkAndFixConsistency();
468 
469     // Check integrity (does not fix)
470     checkIntegrity();
471     return errors.getErrorList().size();
472   }
473 
474   /**
475    * Contacts the master and prints out cluster-wide information
476    * @return 0 on success, non-zero on failure
477    */
478   public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
479     // print hbase server version
480     errors.print("Version: " + status.getHBaseVersion());
481     offlineHdfsIntegrityRepair();
482 
483     // turn the balancer off
484     boolean oldBalancer = admin.setBalancerRunning(false, true);
485     try {
486       onlineConsistencyRepair();
487     }
488     finally {
489       admin.setBalancerRunning(oldBalancer, false);
490     }
491 
492     if (checkRegionBoundaries) {
493       checkRegionBoundaries();
494     }
495 
496     offlineReferenceFileRepair();
497 
498     checkAndFixTableLocks();
499 
500     // Print table summary
501     printTableSummary(tablesInfo);
502     return errors.summarize();
503   }
504 
505   public static byte[] keyOnly (byte[] b) {
506     if (b == null)
507       return b;
508     int rowlength = Bytes.toShort(b, 0);
509     byte[] result = new byte[rowlength];
510     System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);
511     return result;
512   }
513 
514   private static class RegionBoundariesInformation {
515     public byte [] regionName;
516     public byte [] metaFirstKey;
517     public byte [] metaLastKey;
518     public byte [] storesFirstKey;
519     public byte [] storesLastKey;
520     @Override
521     public String toString () {
522       return "regionName=" + Bytes.toStringBinary(regionName) +
523              "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) +
524              "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) +
525              "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) +
526              "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey);
527     }
528   }
529 
530   public void checkRegionBoundaries() {
531     try {
532       ByteArrayComparator comparator = new ByteArrayComparator();
533       List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), false);
534       final RegionBoundariesInformation currentRegionBoundariesInformation =
535           new RegionBoundariesInformation();
536       Path hbaseRoot = FSUtils.getRootDir(getConf());
537       for (HRegionInfo regionInfo : regions) {
538         Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
539         currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
540         // For each region, get the start and stop key from the META and compare them to the
541         // same information from the Stores.
542         Path path = new Path(tableDir, regionInfo.getEncodedName());
543         FileSystem fs = path.getFileSystem(getConf());
544         FileStatus[] files = fs.listStatus(path);
545         // For all the column families in this region...
546         byte[] storeFirstKey = null;
547         byte[] storeLastKey = null;
548         for (FileStatus file : files) {
549           String fileName = file.getPath().toString();
550           fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
551           if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
552             FileStatus[] storeFiles = fs.listStatus(file.getPath());
553             // For all the stores in this column family.
554             for (FileStatus storeFile : storeFiles) {
555               HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig(
556                   getConf()), getConf());
557               if ((reader.getFirstKey() != null)
558                   && ((storeFirstKey == null) || (comparator.compare(storeFirstKey,
559                       reader.getFirstKey()) > 0))) {
560                 storeFirstKey = reader.getFirstKey();
561               }
562               if ((reader.getLastKey() != null)
563                   && ((storeLastKey == null) || (comparator.compare(storeLastKey,
564                       reader.getLastKey())) < 0)) {
565                 storeLastKey = reader.getLastKey();
566               }
567               reader.close();
568             }
569           }
570         }
571         currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
572         currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
573         currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
574         currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
575         if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
576           currentRegionBoundariesInformation.metaFirstKey = null;
577         if (currentRegionBoundariesInformation.metaLastKey.length == 0)
578           currentRegionBoundariesInformation.metaLastKey = null;
579 
580         // For a region to be correct, we need the META start key to be smaller or equal to the
581         // smallest start key from all the stores, and the start key from the next META entry to
582         // be bigger than the last key from all the current stores. First region start key is null;
583         // Last region end key is null; some regions can be empty and not have any store.
584 
585         boolean valid = true;
586         // Checking start key.
587         if ((currentRegionBoundariesInformation.storesFirstKey != null)
588             && (currentRegionBoundariesInformation.metaFirstKey != null)) {
589           valid = valid
590               && comparator.compare(currentRegionBoundariesInformation.storesFirstKey,
591                 currentRegionBoundariesInformation.metaFirstKey) >= 0;
592         }
593         // Checking stop key.
594         if ((currentRegionBoundariesInformation.storesLastKey != null)
595             && (currentRegionBoundariesInformation.metaLastKey != null)) {
596           valid = valid
597               && comparator.compare(currentRegionBoundariesInformation.storesLastKey,
598                 currentRegionBoundariesInformation.metaLastKey) < 0;
599         }
600         if (!valid) {
601           errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",
602             tablesInfo.get(regionInfo.getTable()));
603           LOG.warn("Region's boundaries not alligned between stores and META for:");
604           LOG.warn(currentRegionBoundariesInformation);
605         }
606       }
607     } catch (IOException e) {
608       LOG.error(e);
609     }
610   }
611 
612   /**
613    * Iterates through the list of all orphan/invalid regiondirs.
614    */
615   private void adoptHdfsOrphans(Collection<HbckInfo> orphanHdfsDirs) throws IOException {
616     for (HbckInfo hi : orphanHdfsDirs) {
617       LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());
618       adoptHdfsOrphan(hi);
619     }
620   }
621 
622   /**
623    * Orphaned regions are regions without a .regioninfo file in them.  We "adopt"
624    * these orphans by creating a new region, and moving the column families,
625    * recovered edits, HLogs, into the new region dir.  We determine the region
626    * startkey and endkeys by looking at all of the hfiles inside the column
627    * families to identify the min and max keys. The resulting region will
628    * likely violate table integrity but will be dealt with by merging
629    * overlapping regions.
630    */
631   @SuppressWarnings("deprecation")
632   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
633     Path p = hi.getHdfsRegionDir();
634     FileSystem fs = p.getFileSystem(getConf());
635     FileStatus[] dirs = fs.listStatus(p);
636     if (dirs == null) {
637       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
638           p + ". This dir could probably be deleted.");
639       return ;
640     }
641 
642     TableName tableName = hi.getTableName();
643     TableInfo tableInfo = tablesInfo.get(tableName);
644     Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");
645     HTableDescriptor template = tableInfo.getHTD();
646 
647     // find min and max key values
648     Pair<byte[],byte[]> orphanRegionRange = null;
649     for (FileStatus cf : dirs) {
650       String cfName= cf.getPath().getName();
651       // TODO Figure out what the special dirs are
652       if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;
653 
654       FileStatus[] hfiles = fs.listStatus(cf.getPath());
655       for (FileStatus hfile : hfiles) {
656         byte[] start, end;
657         HFile.Reader hf = null;
658         try {
659           CacheConfig cacheConf = new CacheConfig(getConf());
660           hf = HFile.createReader(fs, hfile.getPath(), cacheConf, getConf());
661           hf.loadFileInfo();
662           KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
663           start = startKv.getRow();
664           KeyValue endKv = KeyValue.createKeyValueFromKey(hf.getLastKey());
665           end = endKv.getRow();
666         } catch (IOException ioe) {
667           LOG.warn("Problem reading orphan file " + hfile + ", skipping");
668           continue;
669         } catch (NullPointerException ioe) {
670           LOG.warn("Orphan file " + hfile + " is possibly corrupted HFile, skipping");
671           continue;
672         } finally {
673           if (hf != null) {
674             hf.close();
675           }
676         }
677 
678         // expand the range to include the range of all hfiles
679         if (orphanRegionRange == null) {
680           // first range
681           orphanRegionRange = new Pair<byte[], byte[]>(start, end);
682         } else {
683           // TODO add test
684 
685           // expand range only if the hfile is wider.
686           if (Bytes.compareTo(orphanRegionRange.getFirst(), start) > 0) {
687             orphanRegionRange.setFirst(start);
688           }
689           if (Bytes.compareTo(orphanRegionRange.getSecond(), end) < 0 ) {
690             orphanRegionRange.setSecond(end);
691           }
692         }
693       }
694     }
695     if (orphanRegionRange == null) {
696       LOG.warn("No data in dir " + p + ", sidelining data");
697       fixes++;
698       sidelineRegionDir(fs, hi);
699       return;
700     }
701     LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", " +
702         Bytes.toString(orphanRegionRange.getSecond()) + ")");
703 
704     // create new region on hdfs.  move data into place.
705     HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond());
706     LOG.info("Creating new region : " + hri);
707     HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
708     Path target = region.getRegionFileSystem().getRegionDir();
709 
710     // rename all the data to new region
711     mergeRegionDirs(target, hi);
712     fixes++;
713   }
714 
715   /**
716    * This method determines if there are table integrity errors in HDFS.  If
717    * there are errors and the appropriate "fix" options are enabled, the method
718    * will first correct orphan regions making them into legit regiondirs, and
719    * then reload to merge potentially overlapping regions.
720    *
721    * @return number of table integrity errors found
722    */
723   private int restoreHdfsIntegrity() throws IOException, InterruptedException {
724     // Determine what's on HDFS
725     LOG.info("Loading HBase regioninfo from HDFS...");
726     loadHdfsRegionDirs(); // populating regioninfo table.
727 
728     int errs = errors.getErrorList().size();
729     // First time just get suggestions.
730     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
731     checkHdfsIntegrity(false, false);
732 
733     if (errors.getErrorList().size() == errs) {
734       LOG.info("No integrity errors.  We are done with this phase. Glorious.");
735       return 0;
736     }
737 
738     if (shouldFixHdfsOrphans() && orphanHdfsDirs.size() > 0) {
739       adoptHdfsOrphans(orphanHdfsDirs);
740       // TODO optimize by incrementally adding instead of reloading.
741     }
742 
743     // Make sure there are no holes now.
744     if (shouldFixHdfsHoles()) {
745       clearState(); // this also resets # fixes.
746       loadHdfsRegionDirs();
747       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
748       tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);
749     }
750 
751     // Now we fix overlaps
752     if (shouldFixHdfsOverlaps()) {
753       // second pass we fix overlaps.
754       clearState(); // this also resets # fixes.
755       loadHdfsRegionDirs();
756       tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
757       tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());
758     }
759 
760     return errors.getErrorList().size();
761   }
762 
763   /**
764    * Scan all the store file names to find any lingering reference files,
765    * which refer to some none-exiting files. If "fix" option is enabled,
766    * any lingering reference file will be sidelined if found.
767    * <p>
768    * Lingering reference file prevents a region from opening. It has to
769    * be fixed before a cluster can start properly.
770    */
771   private void offlineReferenceFileRepair() throws IOException {
772     Configuration conf = getConf();
773     Path hbaseRoot = FSUtils.getRootDir(conf);
774     FileSystem fs = hbaseRoot.getFileSystem(conf);
775     Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot);
776     for (Path path: allFiles.values()) {
777       boolean isReference = false;
778       try {
779         isReference = StoreFileInfo.isReference(path);
780       } catch (Throwable t) {
781         // Ignore. Some files may not be store files at all.
782         // For example, files under .oldlogs folder in hbase:meta
783         // Warning message is already logged by
784         // StoreFile#isReference.
785       }
786       if (!isReference) continue;
787 
788       Path referredToFile = StoreFileInfo.getReferredToFile(path);
789       if (fs.exists(referredToFile)) continue;  // good, expected
790 
791       // Found a lingering reference file
792       errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
793         "Found lingering reference file " + path);
794       if (!shouldFixReferenceFiles()) continue;
795 
796       // Now, trying to fix it since requested
797       boolean success = false;
798       String pathStr = path.toString();
799 
800       // A reference file path should be like
801       // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name
802       // Up 5 directories to get the root folder.
803       // So the file will be sidelined to a similar folder structure.
804       int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
805       for (int i = 0; index > 0 && i < 5; i++) {
806         index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);
807       }
808       if (index > 0) {
809         Path rootDir = getSidelineDir();
810         Path dst = new Path(rootDir, pathStr.substring(index + 1));
811         fs.mkdirs(dst.getParent());
812         LOG.info("Trying to sildeline reference file "
813           + path + " to " + dst);
814         setShouldRerun();
815 
816         success = fs.rename(path, dst);
817       }
818       if (!success) {
819         LOG.error("Failed to sideline reference file " + path);
820       }
821     }
822   }
823 
824   /**
825    * TODO -- need to add tests for this.
826    */
827   private void reportEmptyMetaCells() {
828     errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: " +
829       emptyRegionInfoQualifiers.size());
830     if (details) {
831       for (Result r: emptyRegionInfoQualifiers) {
832         errors.print("  " + r);
833       }
834     }
835   }
836 
837   /**
838    * TODO -- need to add tests for this.
839    */
840   private void reportTablesInFlux() {
841     AtomicInteger numSkipped = new AtomicInteger(0);
842     HTableDescriptor[] allTables = getTables(numSkipped);
843     errors.print("Number of Tables: " + allTables.length);
844     if (details) {
845       if (numSkipped.get() > 0) {
846         errors.detail("Number of Tables in flux: " + numSkipped.get());
847       }
848       for (HTableDescriptor td : allTables) {
849         errors.detail("  Table: " + td.getTableName() + "\t" +
850                            (td.isReadOnly() ? "ro" : "rw") + "\t" +
851                             (td.isMetaRegion() ? "META" : "    ") + "\t" +
852                            " families: " + td.getFamilies().size());
853       }
854     }
855   }
856 
857   public ErrorReporter getErrors() {
858     return errors;
859   }
860 
861   /**
862    * Read the .regioninfo file from the file system.  If there is no
863    * .regioninfo, add it to the orphan hdfs region list.
864    */
865   private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException {
866     Path regionDir = hbi.getHdfsRegionDir();
867     if (regionDir == null) {
868       LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry);
869       return;
870     }
871 
872     if (hbi.hdfsEntry.hri != null) {
873       // already loaded data
874       return;
875     }
876 
877     FileSystem fs = FileSystem.get(getConf());
878     HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
879     LOG.debug("HRegionInfo read: " + hri.toString());
880     hbi.hdfsEntry.hri = hri;
881   }
882 
883   /**
884    * Exception thrown when a integrity repair operation fails in an
885    * unresolvable way.
886    */
887   public static class RegionRepairException extends IOException {
888     private static final long serialVersionUID = 1L;
889     final IOException ioe;
890     public RegionRepairException(String s, IOException ioe) {
891       super(s);
892       this.ioe = ioe;
893     }
894   }
895 
896   /**
897    * Populate hbi's from regionInfos loaded from file system.
898    */
899   private SortedMap<TableName, TableInfo> loadHdfsRegionInfos()
900       throws IOException, InterruptedException {
901     tablesInfo.clear(); // regenerating the data
902     // generate region split structure
903     Collection<HbckInfo> hbckInfos = regionInfoMap.values();
904 
905     // Parallelized read of .regioninfo files.
906     List<WorkItemHdfsRegionInfo> hbis = new ArrayList<WorkItemHdfsRegionInfo>(hbckInfos.size());
907     List<Future<Void>> hbiFutures;
908 
909     for (HbckInfo hbi : hbckInfos) {
910       WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);
911       hbis.add(work);
912     }
913 
914     // Submit and wait for completion
915     hbiFutures = executor.invokeAll(hbis);
916 
917     for(int i=0; i<hbiFutures.size(); i++) {
918       WorkItemHdfsRegionInfo work = hbis.get(i);
919       Future<Void> f = hbiFutures.get(i);
920       try {
921         f.get();
922       } catch(ExecutionException e) {
923         LOG.warn("Failed to read .regioninfo file for region " +
924               work.hbi.getRegionNameAsString(), e.getCause());
925       }
926     }
927 
928     Path hbaseRoot = FSUtils.getRootDir(getConf());
929     FileSystem fs = hbaseRoot.getFileSystem(getConf());
930     // serialized table info gathering.
931     for (HbckInfo hbi: hbckInfos) {
932 
933       if (hbi.getHdfsHRI() == null) {
934         // was an orphan
935         continue;
936       }
937 
938 
939       // get table name from hdfs, populate various HBaseFsck tables.
940       TableName tableName = hbi.getTableName();
941       if (tableName == null) {
942         // There was an entry in hbase:meta not in the HDFS?
943         LOG.warn("tableName was null for: " + hbi);
944         continue;
945       }
946 
947       TableInfo modTInfo = tablesInfo.get(tableName);
948       if (modTInfo == null) {
949         // only executed once per table.
950         modTInfo = new TableInfo(tableName);
951         tablesInfo.put(tableName, modTInfo);
952         try {
953           HTableDescriptor htd =
954               FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);
955           modTInfo.htds.add(htd);
956         } catch (IOException ioe) {
957           if (!orphanTableDirs.containsKey(tableName)) {
958             LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);
959             //should only report once for each table
960             errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,
961                 "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);
962             Set<String> columns = new HashSet<String>();
963             orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));
964           }
965         }
966       }
967       if (!hbi.isSkipChecks()) {
968         modTInfo.addRegionInfo(hbi);
969       }
970     }
971 
972     loadTableInfosForTablesWithNoRegion();
973 
974     return tablesInfo;
975   }
976 
977   /**
978    * To get the column family list according to the column family dirs
979    * @param columns
980    * @param hbi
981    * @return a set of column families
982    * @throws IOException
983    */
984   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
985     Path regionDir = hbi.getHdfsRegionDir();
986     FileSystem fs = regionDir.getFileSystem(getConf());
987     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
988     for (FileStatus subdir : subDirs) {
989       String columnfamily = subdir.getPath().getName();
990       columns.add(columnfamily);
991     }
992     return columns;
993   }
994 
995   /**
996    * To fabricate a .tableinfo file with following contents<br>
997    * 1. the correct tablename <br>
998    * 2. the correct colfamily list<br>
999    * 3. the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1000    * @throws IOException
1001    */
1002   private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,
1003       Set<String> columns) throws IOException {
1004     if (columns ==null || columns.isEmpty()) return false;
1005     HTableDescriptor htd = new HTableDescriptor(tableName);
1006     for (String columnfamimly : columns) {
1007       htd.addFamily(new HColumnDescriptor(columnfamimly));
1008     }
1009     fstd.createTableDescriptor(htd, true);
1010     return true;
1011   }
1012 
1013   /**
1014    * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta <br>
1015    * @throws IOException
1016    */
1017   public void fixEmptyMetaCells() throws IOException {
1018     if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
1019       LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");
1020       for (Result region : emptyRegionInfoQualifiers) {
1021         deleteMetaRegion(region.getRow());
1022         errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
1023       }
1024       emptyRegionInfoQualifiers.clear();
1025     }
1026   }
1027 
1028   /**
1029    * To fix orphan table by creating a .tableinfo file under tableDir <br>
1030    * 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
1031    * 2. else create a default .tableinfo file with following items<br>
1032    * &nbsp;2.1 the correct tablename <br>
1033    * &nbsp;2.2 the correct colfamily list<br>
1034    * &nbsp;2.3 the default properties for both {@link HTableDescriptor} and {@link HColumnDescriptor}<br>
1035    * @throws IOException
1036    */
1037   public void fixOrphanTables() throws IOException {
1038     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
1039 
1040       List<TableName> tmpList = new ArrayList<TableName>();
1041       tmpList.addAll(orphanTableDirs.keySet());
1042       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
1043       Iterator<Entry<TableName, Set<String>>> iter =
1044           orphanTableDirs.entrySet().iterator();
1045       int j = 0;
1046       int numFailedCase = 0;
1047       FSTableDescriptors fstd = new FSTableDescriptors(getConf());
1048       while (iter.hasNext()) {
1049         Entry<TableName, Set<String>> entry =
1050             (Entry<TableName, Set<String>>) iter.next();
1051         TableName tableName = entry.getKey();
1052         LOG.info("Trying to fix orphan table error: " + tableName);
1053         if (j < htds.length) {
1054           if (tableName.equals(htds[j].getTableName())) {
1055             HTableDescriptor htd = htds[j];
1056             LOG.info("fixing orphan table: " + tableName + " from cache");
1057             fstd.createTableDescriptor(htd, true);
1058             j++;
1059             iter.remove();
1060           }
1061         } else {
1062           if (fabricateTableInfo(fstd, tableName, entry.getValue())) {
1063             LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");
1064             LOG.warn("Strongly recommend to modify the HTableDescriptor if necessary for: " + tableName);
1065             iter.remove();
1066           } else {
1067             LOG.error("Unable to create default .tableinfo for " + tableName + " while missing column family information");
1068             numFailedCase++;
1069           }
1070         }
1071         fixes++;
1072       }
1073 
1074       if (orphanTableDirs.isEmpty()) {
1075         // all orphanTableDirs are luckily recovered
1076         // re-run doFsck after recovering the .tableinfo file
1077         setShouldRerun();
1078         LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");
1079       } else if (numFailedCase > 0) {
1080         LOG.error("Failed to fix " + numFailedCase
1081             + " OrphanTables with default .tableinfo files");
1082       }
1083 
1084     }
1085     //cleanup the list
1086     orphanTableDirs.clear();
1087 
1088   }
1089 
1090   /**
1091    * This borrows code from MasterFileSystem.bootstrap()
1092    *
1093    * @return an open hbase:meta HRegion
1094    */
1095   private HRegion createNewMeta() throws IOException {
1096       Path rootdir = FSUtils.getRootDir(getConf());
1097     Configuration c = getConf();
1098     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
1099     MasterFileSystem.setInfoFamilyCachingForMeta(false);
1100     HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c,
1101         HTableDescriptor.META_TABLEDESC);
1102     MasterFileSystem.setInfoFamilyCachingForMeta(true);
1103     return meta;
1104   }
1105 
1106   /**
1107    * Generate set of puts to add to new meta.  This expects the tables to be
1108    * clean with no overlaps or holes.  If there are any problems it returns null.
1109    *
1110    * @return An array list of puts to do in bulk, null if tables have problems
1111    */
1112   private ArrayList<Put> generatePuts(
1113       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1114     ArrayList<Put> puts = new ArrayList<Put>();
1115     boolean hasProblems = false;
1116     for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
1117       TableName name = e.getKey();
1118 
1119       // skip "hbase:meta"
1120       if (name.compareTo(TableName.META_TABLE_NAME) == 0) {
1121         continue;
1122       }
1123 
1124       TableInfo ti = e.getValue();
1125       for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
1126           .entrySet()) {
1127         Collection<HbckInfo> his = spl.getValue();
1128         int sz = his.size();
1129         if (sz != 1) {
1130           // problem
1131           LOG.error("Split starting at " + Bytes.toStringBinary(spl.getKey())
1132               + " had " +  sz + " regions instead of exactly 1." );
1133           hasProblems = true;
1134           continue;
1135         }
1136 
1137         // add the row directly to meta.
1138         HbckInfo hi = his.iterator().next();
1139         HRegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
1140         Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1141         puts.add(p);
1142       }
1143     }
1144     return hasProblems ? null : puts;
1145   }
1146 
1147   /**
1148    * Suggest fixes for each table
1149    */
1150   private void suggestFixes(
1151       SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
1152     for (TableInfo tInfo : tablesInfo.values()) {
1153       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1154       tInfo.checkRegionChain(handler);
1155     }
1156   }
1157 
1158   /**
1159    * Rebuilds meta from information in hdfs/fs.  Depends on configuration
1160    * settings passed into hbck constructor to point to a particular fs/dir.
1161    *
1162    * @param fix flag that determines if method should attempt to fix holes
1163    * @return true if successful, false if attempt failed.
1164    */
1165   public boolean rebuildMeta(boolean fix) throws IOException,
1166       InterruptedException {
1167 
1168     // TODO check to make sure hbase is offline. (or at least the table
1169     // currently being worked on is off line)
1170 
1171     // Determine what's on HDFS
1172     LOG.info("Loading HBase regioninfo from HDFS...");
1173     loadHdfsRegionDirs(); // populating regioninfo table.
1174 
1175     int errs = errors.getErrorList().size();
1176     tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1177     checkHdfsIntegrity(false, false);
1178 
1179     // make sure ok.
1180     if (errors.getErrorList().size() != errs) {
1181       // While in error state, iterate until no more fixes possible
1182       while(true) {
1183         fixes = 0;
1184         suggestFixes(tablesInfo);
1185         errors.clear();
1186         loadHdfsRegionInfos(); // update tableInfos based on region info in fs.
1187         checkHdfsIntegrity(shouldFixHdfsHoles(), shouldFixHdfsOverlaps());
1188 
1189         int errCount = errors.getErrorList().size();
1190 
1191         if (fixes == 0) {
1192           if (errCount > 0) {
1193             return false; // failed to fix problems.
1194           } else {
1195             break; // no fixes and no problems? drop out and fix stuff!
1196           }
1197         }
1198       }
1199     }
1200 
1201     // we can rebuild, move old meta out of the way and start
1202     LOG.info("HDFS regioninfo's seems good.  Sidelining old hbase:meta");
1203     Path backupDir = sidelineOldMeta();
1204 
1205     LOG.info("Creating new hbase:meta");
1206     HRegion meta = createNewMeta();
1207 
1208     // populate meta
1209     List<Put> puts = generatePuts(tablesInfo);
1210     if (puts == null) {
1211       LOG.fatal("Problem encountered when creating new hbase:meta entries.  " +
1212         "You may need to restore the previously sidelined hbase:meta");
1213       return false;
1214     }
1215     meta.batchMutate(puts.toArray(new Put[0]));
1216     HRegion.closeHRegion(meta);
1217     LOG.info("Success! hbase:meta table rebuilt.");
1218     LOG.info("Old hbase:meta is moved into " + backupDir);
1219     return true;
1220   }
1221 
1222   private SortedMap<TableName, TableInfo> checkHdfsIntegrity(boolean fixHoles,
1223       boolean fixOverlaps) throws IOException {
1224     LOG.info("Checking HBase region split map from HDFS data...");
1225     for (TableInfo tInfo : tablesInfo.values()) {
1226       TableIntegrityErrorHandler handler;
1227       if (fixHoles || fixOverlaps) {
1228         handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
1229           fixHoles, fixOverlaps);
1230       } else {
1231         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1232       }
1233       if (!tInfo.checkRegionChain(handler)) {
1234         // should dump info as well.
1235         errors.report("Found inconsistency in table " + tInfo.getName());
1236       }
1237     }
1238     return tablesInfo;
1239   }
1240 
1241   private Path getSidelineDir() throws IOException {
1242     if (sidelineDir == null) {
1243       Path hbaseDir = FSUtils.getRootDir(getConf());
1244       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
1245       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
1246           + startMillis);
1247     }
1248     return sidelineDir;
1249   }
1250 
1251   /**
1252    * Sideline a region dir (instead of deleting it)
1253    */
1254   Path sidelineRegionDir(FileSystem fs, HbckInfo hi) throws IOException {
1255     return sidelineRegionDir(fs, null, hi);
1256   }
1257 
1258   /**
1259    * Sideline a region dir (instead of deleting it)
1260    *
1261    * @param parentDir if specified, the region will be sidelined to
1262    * folder like .../parentDir/<table name>/<region name>. The purpose
1263    * is to group together similar regions sidelined, for example, those
1264    * regions should be bulk loaded back later on. If null, it is ignored.
1265    */
1266   Path sidelineRegionDir(FileSystem fs,
1267       String parentDir, HbckInfo hi) throws IOException {
1268     TableName tableName = hi.getTableName();
1269     Path regionDir = hi.getHdfsRegionDir();
1270 
1271     if (!fs.exists(regionDir)) {
1272       LOG.warn("No previous " + regionDir + " exists.  Continuing.");
1273       return null;
1274     }
1275 
1276     Path rootDir = getSidelineDir();
1277     if (parentDir != null) {
1278       rootDir = new Path(rootDir, parentDir);
1279     }
1280     Path sidelineTableDir= FSUtils.getTableDir(rootDir, tableName);
1281     Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());
1282     fs.mkdirs(sidelineRegionDir);
1283     boolean success = false;
1284     FileStatus[] cfs =  fs.listStatus(regionDir);
1285     if (cfs == null) {
1286       LOG.info("Region dir is empty: " + regionDir);
1287     } else {
1288       for (FileStatus cf : cfs) {
1289         Path src = cf.getPath();
1290         Path dst =  new Path(sidelineRegionDir, src.getName());
1291         if (fs.isFile(src)) {
1292           // simple file
1293           success = fs.rename(src, dst);
1294           if (!success) {
1295             String msg = "Unable to rename file " + src +  " to " + dst;
1296             LOG.error(msg);
1297             throw new IOException(msg);
1298           }
1299           continue;
1300         }
1301 
1302         // is a directory.
1303         fs.mkdirs(dst);
1304 
1305         LOG.info("Sidelining files from " + src + " into containing region " + dst);
1306         // FileSystem.rename is inconsistent with directories -- if the
1307         // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
1308         // it moves the src into the dst dir resulting in (foo/a/b).  If
1309         // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
1310         FileStatus[] hfiles = fs.listStatus(src);
1311         if (hfiles != null && hfiles.length > 0) {
1312           for (FileStatus hfile : hfiles) {
1313             success = fs.rename(hfile.getPath(), dst);
1314             if (!success) {
1315               String msg = "Unable to rename file " + src +  " to " + dst;
1316               LOG.error(msg);
1317               throw new IOException(msg);
1318             }
1319           }
1320         }
1321         LOG.debug("Sideline directory contents:");
1322         debugLsr(sidelineRegionDir);
1323       }
1324     }
1325 
1326     LOG.info("Removing old region dir: " + regionDir);
1327     success = fs.delete(regionDir, true);
1328     if (!success) {
1329       String msg = "Unable to delete dir " + regionDir;
1330       LOG.error(msg);
1331       throw new IOException(msg);
1332     }
1333     return sidelineRegionDir;
1334   }
1335 
1336   /**
1337    * Side line an entire table.
1338    */
1339   void sidelineTable(FileSystem fs, TableName tableName, Path hbaseDir,
1340       Path backupHbaseDir) throws IOException {
1341     Path tableDir = FSUtils.getTableDir(hbaseDir, tableName);
1342     if (fs.exists(tableDir)) {
1343       Path backupTableDir= FSUtils.getTableDir(backupHbaseDir, tableName);
1344       fs.mkdirs(backupTableDir.getParent());
1345       boolean success = fs.rename(tableDir, backupTableDir);
1346       if (!success) {
1347         throw new IOException("Failed to move  " + tableName + " from "
1348             +  tableDir + " to " + backupTableDir);
1349       }
1350     } else {
1351       LOG.info("No previous " + tableName +  " exists.  Continuing.");
1352     }
1353   }
1354 
1355   /**
1356    * @return Path to backup of original directory
1357    */
1358   Path sidelineOldMeta() throws IOException {
1359     // put current hbase:meta aside.
1360     Path hbaseDir = FSUtils.getRootDir(getConf());
1361     FileSystem fs = hbaseDir.getFileSystem(getConf());
1362     Path backupDir = getSidelineDir();
1363     fs.mkdirs(backupDir);
1364 
1365     try {
1366       sidelineTable(fs, TableName.META_TABLE_NAME, hbaseDir, backupDir);
1367     } catch (IOException e) {
1368         LOG.fatal("... failed to sideline meta. Currently in inconsistent state.  To restore "
1369             + "try to rename hbase:meta in " + backupDir.getName() + " to "
1370             + hbaseDir.getName() + ".", e);
1371       throw e; // throw original exception
1372     }
1373     return backupDir;
1374   }
1375 
1376   /**
1377    * Load the list of disabled tables in ZK into local set.
1378    * @throws ZooKeeperConnectionException
1379    * @throws IOException
1380    */
1381   private void loadDisabledTables()
1382   throws ZooKeeperConnectionException, IOException {
1383     HConnectionManager.execute(new HConnectable<Void>(getConf()) {
1384       @Override
1385       public Void connect(HConnection connection) throws IOException {
1386         ZooKeeperWatcher zkw = createZooKeeperWatcher();
1387         try {
1388           for (TableName tableName :
1389               ZKTableStateClientSideReader.getDisabledOrDisablingTables(zkw)) {
1390             disabledTables.add(tableName);
1391           }
1392         } catch (KeeperException ke) {
1393           throw new IOException(ke);
1394         } catch (InterruptedException e) {
1395           throw new InterruptedIOException();
1396         } finally {
1397           zkw.close();
1398         }
1399         return null;
1400       }
1401     });
1402   }
1403 
1404   /**
1405    * Check if the specified region's table is disabled.
1406    */
1407   private boolean isTableDisabled(HRegionInfo regionInfo) {
1408     return disabledTables.contains(regionInfo.getTable());
1409   }
1410 
1411   /**
1412    * Scan HDFS for all regions, recording their information into
1413    * regionInfoMap
1414    */
1415   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
1416     Path rootDir = FSUtils.getRootDir(getConf());
1417     FileSystem fs = rootDir.getFileSystem(getConf());
1418 
1419     // list all tables from HDFS
1420     List<FileStatus> tableDirs = Lists.newArrayList();
1421 
1422     boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));
1423 
1424     List<Path> paths = FSUtils.getTableDirs(fs, rootDir);
1425     for (Path path : paths) {
1426       TableName tableName = FSUtils.getTableName(path);
1427        if ((!checkMetaOnly &&
1428            isTableIncluded(tableName)) ||
1429            tableName.equals(TableName.META_TABLE_NAME)) {
1430          tableDirs.add(fs.getFileStatus(path));
1431        }
1432     }
1433 
1434     // verify that version file exists
1435     if (!foundVersionFile) {
1436       errors.reportError(ERROR_CODE.NO_VERSION_FILE,
1437           "Version file does not exist in root dir " + rootDir);
1438       if (shouldFixVersionFile()) {
1439         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
1440             + " file.");
1441         setShouldRerun();
1442         FSUtils.setVersion(fs, rootDir, getConf().getInt(
1443             HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
1444             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
1445             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
1446       }
1447     }
1448 
1449     // level 1:  <HBASE_DIR>/*
1450     List<WorkItemHdfsDir> dirs = new ArrayList<WorkItemHdfsDir>(tableDirs.size());
1451     List<Future<Void>> dirsFutures;
1452 
1453     for (FileStatus tableDir : tableDirs) {
1454       LOG.debug("Loading region dirs from " +tableDir.getPath());
1455       dirs.add(new WorkItemHdfsDir(this, fs, errors, tableDir));
1456     }
1457 
1458     // Invoke and wait for Callables to complete
1459     dirsFutures = executor.invokeAll(dirs);
1460 
1461     for(Future<Void> f: dirsFutures) {
1462       try {
1463         f.get();
1464       } catch(ExecutionException e) {
1465         LOG.warn("Could not load region dir " , e.getCause());
1466       }
1467     }
1468   }
1469 
1470   /**
1471    * Record the location of the hbase:meta region as found in ZooKeeper.
1472    */
1473   private boolean recordMetaRegion() throws IOException {
1474     HRegionLocation metaLocation = connection.locateRegion(
1475       TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW);
1476 
1477     // Check if Meta region is valid and existing
1478     if (metaLocation == null || metaLocation.getRegionInfo() == null ||
1479         metaLocation.getHostname() == null) {
1480       errors.reportError(ERROR_CODE.NULL_META_REGION,
1481         "META region or some of its attributes are null.");
1482       return false;
1483     }
1484     ServerName sn;
1485     try {
1486       sn = getMetaRegionServerName();
1487     } catch (KeeperException e) {
1488       throw new IOException(e);
1489     }
1490     MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, System.currentTimeMillis());
1491     HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName());
1492     if (hbckInfo == null) {
1493       regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m));
1494     } else {
1495       hbckInfo.metaEntry = m;
1496     }
1497     return true;
1498   }
1499 
1500   private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
1501     return new ZooKeeperWatcher(getConf(), "hbase Fsck", new Abortable() {
1502       @Override
1503       public void abort(String why, Throwable e) {
1504         LOG.error(why, e);
1505         System.exit(1);
1506       }
1507 
1508       @Override
1509       public boolean isAborted() {
1510         return false;
1511       }
1512 
1513     });
1514   }
1515 
1516   private ServerName getMetaRegionServerName()
1517   throws IOException, KeeperException {
1518     ZooKeeperWatcher zkw = createZooKeeperWatcher();
1519     ServerName sn = null;
1520     try {
1521       sn = new MetaTableLocator().getMetaRegionLocation(zkw);
1522     } finally {
1523       zkw.close();
1524     }
1525     return sn;
1526   }
1527 
1528   /**
1529    * Contacts each regionserver and fetches metadata about regions.
1530    * @param regionServerList - the list of region servers to connect to
1531    * @throws IOException if a remote or network exception occurs
1532    */
1533   void processRegionServers(Collection<ServerName> regionServerList)
1534     throws IOException, InterruptedException {
1535 
1536     List<WorkItemRegion> workItems = new ArrayList<WorkItemRegion>(regionServerList.size());
1537     List<Future<Void>> workFutures;
1538 
1539     // loop to contact each region server in parallel
1540     for (ServerName rsinfo: regionServerList) {
1541       workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
1542     }
1543 
1544     workFutures = executor.invokeAll(workItems);
1545 
1546     for(int i=0; i<workFutures.size(); i++) {
1547       WorkItemRegion item = workItems.get(i);
1548       Future<Void> f = workFutures.get(i);
1549       try {
1550         f.get();
1551       } catch(ExecutionException e) {
1552         LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(),
1553             e.getCause());
1554       }
1555     }
1556   }
1557 
1558   /**
1559    * Check consistency of all regions that have been found in previous phases.
1560    */
1561   private void checkAndFixConsistency()
1562   throws IOException, KeeperException, InterruptedException {
1563     for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) {
1564       checkRegionConsistency(e.getKey(), e.getValue());
1565     }
1566   }
1567 
1568   private void preCheckPermission() throws IOException, AccessControlException {
1569     if (shouldIgnorePreCheckPermission()) {
1570       return;
1571     }
1572 
1573     Path hbaseDir = FSUtils.getRootDir(getConf());
1574     FileSystem fs = hbaseDir.getFileSystem(getConf());
1575     UserProvider userProvider = UserProvider.instantiate(getConf());
1576     UserGroupInformation ugi = userProvider.getCurrent().getUGI();
1577     FileStatus[] files = fs.listStatus(hbaseDir);
1578     for (FileStatus file : files) {
1579       try {
1580         FSUtils.checkAccess(ugi, file, FsAction.WRITE);
1581       } catch (AccessControlException ace) {
1582         LOG.warn("Got AccessControlException when preCheckPermission ", ace);
1583         errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName()
1584           + " does not have write perms to " + file.getPath()
1585           + ". Please rerun hbck as hdfs user " + file.getOwner());
1586         throw new AccessControlException(ace);
1587       }
1588     }
1589   }
1590 
1591   /**
1592    * Deletes region from meta table
1593    */
1594   private void deleteMetaRegion(HbckInfo hi) throws IOException {
1595     deleteMetaRegion(hi.metaEntry.getRegionName());
1596   }
1597 
1598   /**
1599    * Deletes region from meta table
1600    */
1601   private void deleteMetaRegion(byte[] metaKey) throws IOException {
1602     Delete d = new Delete(metaKey);
1603     meta.delete(d);
1604     meta.flushCommits();
1605     LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
1606   }
1607 
1608   /**
1609    * Reset the split parent region info in meta table
1610    */
1611   private void resetSplitParent(HbckInfo hi) throws IOException {
1612     RowMutations mutations = new RowMutations(hi.metaEntry.getRegionName());
1613     Delete d = new Delete(hi.metaEntry.getRegionName());
1614     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
1615     d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
1616     mutations.add(d);
1617 
1618     HRegionInfo hri = new HRegionInfo(hi.metaEntry);
1619     hri.setOffline(false);
1620     hri.setSplit(false);
1621     Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
1622     mutations.add(p);
1623 
1624     meta.mutateRow(mutations);
1625     meta.flushCommits();
1626     LOG.info("Reset split parent " + hi.metaEntry.getRegionNameAsString() + " in META" );
1627   }
1628 
1629   /**
1630    * This backwards-compatibility wrapper for permanently offlining a region
1631    * that should not be alive.  If the region server does not support the
1632    * "offline" method, it will use the closest unassign method instead.  This
1633    * will basically work until one attempts to disable or delete the affected
1634    * table.  The problem has to do with in-memory only master state, so
1635    * restarting the HMaster or failing over to another should fix this.
1636    */
1637   private void offline(byte[] regionName) throws IOException {
1638     String regionString = Bytes.toStringBinary(regionName);
1639     if (!rsSupportsOffline) {
1640       LOG.warn("Using unassign region " + regionString
1641           + " instead of using offline method, you should"
1642           + " restart HMaster after these repairs");
1643       admin.unassign(regionName, true);
1644       return;
1645     }
1646 
1647     // first time we assume the rs's supports #offline.
1648     try {
1649       LOG.info("Offlining region " + regionString);
1650       admin.offline(regionName);
1651     } catch (IOException ioe) {
1652       String notFoundMsg = "java.lang.NoSuchMethodException: " +
1653         "org.apache.hadoop.hbase.master.HMaster.offline([B)";
1654       if (ioe.getMessage().contains(notFoundMsg)) {
1655         LOG.warn("Using unassign region " + regionString
1656             + " instead of using offline method, you should"
1657             + " restart HMaster after these repairs");
1658         rsSupportsOffline = false; // in the future just use unassign
1659         admin.unassign(regionName, true);
1660         return;
1661       }
1662       throw ioe;
1663     }
1664   }
1665 
1666   private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException {
1667     for (OnlineEntry rse : hi.deployedEntries) {
1668       LOG.debug("Undeploy region "  + rse.hri + " from " + rse.hsa);
1669       try {
1670         HBaseFsckRepair.closeRegionSilentlyAndWait(admin, rse.hsa, rse.hri);
1671         offline(rse.hri.getRegionName());
1672       } catch (IOException ioe) {
1673         LOG.warn("Got exception when attempting to offline region "
1674             + Bytes.toString(rse.hri.getRegionName()), ioe);
1675       }
1676     }
1677   }
1678 
1679   /**
1680    * Attempts to undeploy a region from a region server based in information in
1681    * META.  Any operations that modify the file system should make sure that
1682    * its corresponding region is not deployed to prevent data races.
1683    *
1684    * A separate call is required to update the master in-memory region state
1685    * kept in the AssignementManager.  Because disable uses this state instead of
1686    * that found in META, we can't seem to cleanly disable/delete tables that
1687    * have been hbck fixed.  When used on a version of HBase that does not have
1688    * the offline ipc call exposed on the master (<0.90.5, <0.92.0) a master
1689    * restart or failover may be required.
1690    */
1691   private void closeRegion(HbckInfo hi) throws IOException, InterruptedException {
1692     if (hi.metaEntry == null && hi.hdfsEntry == null) {
1693       undeployRegions(hi);
1694       return;
1695     }
1696 
1697     // get assignment info and hregioninfo from meta.
1698     Get get = new Get(hi.getRegionName());
1699     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1700     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
1701     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
1702     Result r = meta.get(get);
1703     ServerName serverName = HRegionInfo.getServerName(r);
1704     if (serverName == null) {
1705       errors.reportError("Unable to close region "
1706           + hi.getRegionNameAsString() +  " because meta does not "
1707           + "have handle to reach it.");
1708       return;
1709     }
1710 
1711     HRegionInfo hri = HRegionInfo.getHRegionInfo(r);
1712     if (hri == null) {
1713       LOG.warn("Unable to close region " + hi.getRegionNameAsString()
1714           + " because hbase:meta had invalid or missing "
1715           + HConstants.CATALOG_FAMILY_STR + ":"
1716           + Bytes.toString(HConstants.REGIONINFO_QUALIFIER)
1717           + " qualifier value.");
1718       return;
1719     }
1720 
1721     // close the region -- close files and remove assignment
1722     HBaseFsckRepair.closeRegionSilentlyAndWait(admin, serverName, hri);
1723   }
1724 
1725   private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException,
1726     KeeperException, InterruptedException {
1727     // If we are trying to fix the errors
1728     if (shouldFixAssignments()) {
1729       errors.print(msg);
1730       undeployRegions(hbi);
1731       setShouldRerun();
1732       HRegionInfo hri = hbi.getHdfsHRI();
1733       if (hri == null) {
1734         hri = hbi.metaEntry;
1735       }
1736       HBaseFsckRepair.fixUnassigned(admin, hri);
1737       HBaseFsckRepair.waitUntilAssigned(admin, hri);
1738     }
1739   }
1740 
1741   /**
1742    * Check a single region for consistency and correct deployment.
1743    */
1744   private void checkRegionConsistency(final String key, final HbckInfo hbi)
1745   throws IOException, KeeperException, InterruptedException {
1746     String descriptiveName = hbi.toString();
1747 
1748     boolean inMeta = hbi.metaEntry != null;
1749     // In case not checking HDFS, assume the region is on HDFS
1750     boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
1751     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
1752     boolean isDeployed = !hbi.deployedOn.isEmpty();
1753     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
1754     boolean deploymentMatchesMeta =
1755       hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
1756       hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
1757     boolean splitParent =
1758       (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
1759     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
1760     boolean recentlyModified = inHdfs &&
1761       hbi.getModTime() + timelag > System.currentTimeMillis();
1762 
1763     // ========== First the healthy cases =============
1764     if (hbi.containsOnlyHdfsEdits()) {
1765       return;
1766     }
1767     if (hbi.isSkipChecks()) return;
1768     if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
1769       return;
1770     } else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
1771       LOG.info("Region " + descriptiveName + " is in META, and in a disabled " +
1772         "tabled that is not deployed");
1773       return;
1774     } else if (recentlyModified) {
1775       LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");
1776       return;
1777     }
1778     // ========== Cases where the region is not in hbase:meta =============
1779     else if (!inMeta && !inHdfs && !isDeployed) {
1780       // We shouldn't have record of this region at all then!
1781       assert false : "Entry for region with no data";
1782     } else if (!inMeta && !inHdfs && isDeployed) {
1783       errors.reportError(ERROR_CODE.NOT_IN_META_HDFS, "Region "
1784           + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but " +
1785           "deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1786       if (shouldFixAssignments()) {
1787         undeployRegions(hbi);
1788       }
1789 
1790     } else if (!inMeta && inHdfs && !isDeployed) {
1791       if (hbi.isMerged()) {
1792         // This region has already been merged, the remaining hdfs file will be
1793         // cleaned by CatalogJanitor later
1794         hbi.setSkipChecks(true);
1795         LOG.info("Region " + descriptiveName
1796             + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");
1797         return;
1798       }
1799       errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region "
1800           + descriptiveName + " on HDFS, but not listed in hbase:meta " +
1801           "or deployed on any region server");
1802       // restore region consistency of an adopted orphan
1803       if (shouldFixMeta()) {
1804         if (!hbi.isHdfsRegioninfoPresent()) {
1805           LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"
1806               +  " in table integrity repair phase if -fixHdfsOrphans was" +
1807               " used.");
1808           return;
1809         }
1810 
1811         LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());
1812         HBaseFsckRepair.fixMetaHoleOnline(getConf(), hbi.getHdfsHRI());
1813 
1814         tryAssignmentRepair(hbi, "Trying to reassign region...");
1815       }
1816 
1817     } else if (!inMeta && inHdfs && isDeployed) {
1818       errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName
1819           + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1820       debugLsr(hbi.getHdfsRegionDir());
1821       if (shouldFixMeta()) {
1822         if (!hbi.isHdfsRegioninfoPresent()) {
1823           LOG.error("This should have been repaired in table integrity repair phase");
1824           return;
1825         }
1826 
1827         LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());
1828         HBaseFsckRepair.fixMetaHoleOnline(getConf(), hbi.getHdfsHRI());
1829 
1830         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
1831       }
1832 
1833     // ========== Cases where the region is in hbase:meta =============
1834     } else if (inMeta && inHdfs && !isDeployed && splitParent) {
1835       // check whether this is an actual error, or just transient state where parent
1836       // is not cleaned
1837       if (hbi.metaEntry.splitA != null && hbi.metaEntry.splitB != null) {
1838         // check that split daughters are there
1839         HbckInfo infoA = this.regionInfoMap.get(hbi.metaEntry.splitA.getEncodedName());
1840         HbckInfo infoB = this.regionInfoMap.get(hbi.metaEntry.splitB.getEncodedName());
1841         if (infoA != null && infoB != null) {
1842           // we already processed or will process daughters. Move on, nothing to see here.
1843           hbi.setSkipChecks(true);
1844           return;
1845         }
1846       }
1847       errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region "
1848           + descriptiveName + " is a split parent in META, in HDFS, "
1849           + "and not deployed on any region server. This could be transient.");
1850       if (shouldFixSplitParents()) {
1851         setShouldRerun();
1852         resetSplitParent(hbi);
1853       }
1854     } else if (inMeta && !inHdfs && !isDeployed) {
1855       errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region "
1856           + descriptiveName + " found in META, but not in HDFS "
1857           + "or deployed on any region server.");
1858       if (shouldFixMeta()) {
1859         deleteMetaRegion(hbi);
1860       }
1861     } else if (inMeta && !inHdfs && isDeployed) {
1862       errors.reportError(ERROR_CODE.NOT_IN_HDFS, "Region " + descriptiveName
1863           + " found in META, but not in HDFS, " +
1864           "and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1865       // We treat HDFS as ground truth.  Any information in meta is transient
1866       // and equivalent data can be regenerated.  So, lets unassign and remove
1867       // these problems from META.
1868       if (shouldFixAssignments()) {
1869         errors.print("Trying to fix unassigned region...");
1870         undeployRegions(hbi);
1871       }
1872       if (shouldFixMeta()) {
1873         // wait for it to complete
1874         deleteMetaRegion(hbi);
1875       }
1876     } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
1877       errors.reportError(ERROR_CODE.NOT_DEPLOYED, "Region " + descriptiveName
1878           + " not deployed on any region server.");
1879       tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
1880     } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
1881       errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,
1882           "Region " + descriptiveName + " should not be deployed according " +
1883           "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn));
1884       if (shouldFixAssignments()) {
1885         errors.print("Trying to close the region " + descriptiveName);
1886         setShouldRerun();
1887         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
1888       }
1889     } else if (inMeta && inHdfs && isMultiplyDeployed) {
1890       errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName
1891           + " is listed in hbase:meta on region server " + hbi.metaEntry.regionServer
1892           + " but is multiply assigned to region servers " +
1893           Joiner.on(", ").join(hbi.deployedOn));
1894       // If we are trying to fix the errors
1895       if (shouldFixAssignments()) {
1896         errors.print("Trying to fix assignment error...");
1897         setShouldRerun();
1898         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
1899       }
1900     } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
1901       errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region "
1902           + descriptiveName + " listed in hbase:meta on region server " +
1903           hbi.metaEntry.regionServer + " but found on region server " +
1904           hbi.deployedOn.get(0));
1905       // If we are trying to fix the errors
1906       if (shouldFixAssignments()) {
1907         errors.print("Trying to fix assignment error...");
1908         setShouldRerun();
1909         HBaseFsckRepair.fixMultiAssignment(admin, hbi.metaEntry, hbi.deployedOn);
1910         HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
1911       }
1912     } else {
1913       errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName +
1914           " is in an unforeseen state:" +
1915           " inMeta=" + inMeta +
1916           " inHdfs=" + inHdfs +
1917           " isDeployed=" + isDeployed +
1918           " isMultiplyDeployed=" + isMultiplyDeployed +
1919           " deploymentMatchesMeta=" + deploymentMatchesMeta +
1920           " shouldBeDeployed=" + shouldBeDeployed);
1921     }
1922   }
1923 
1924   /**
1925    * Checks tables integrity. Goes over all regions and scans the tables.
1926    * Collects all the pieces for each table and checks if there are missing,
1927    * repeated or overlapping ones.
1928    * @throws IOException
1929    */
1930   SortedMap<TableName, TableInfo> checkIntegrity() throws IOException {
1931     tablesInfo = new TreeMap<TableName,TableInfo> ();
1932     LOG.debug("There are " + regionInfoMap.size() + " region info entries");
1933     for (HbckInfo hbi : regionInfoMap.values()) {
1934       // Check only valid, working regions
1935       if (hbi.metaEntry == null) {
1936         // this assumes that consistency check has run loadMetaEntry
1937         Path p = hbi.getHdfsRegionDir();
1938         if (p == null) {
1939           errors.report("No regioninfo in Meta or HDFS. " + hbi);
1940         }
1941 
1942         // TODO test.
1943         continue;
1944       }
1945       if (hbi.metaEntry.regionServer == null) {
1946         errors.detail("Skipping region because no region server: " + hbi);
1947         continue;
1948       }
1949       if (hbi.metaEntry.isOffline()) {
1950         errors.detail("Skipping region because it is offline: " + hbi);
1951         continue;
1952       }
1953       if (hbi.containsOnlyHdfsEdits()) {
1954         errors.detail("Skipping region because it only contains edits" + hbi);
1955         continue;
1956       }
1957 
1958       // Missing regionDir or over-deployment is checked elsewhere. Include
1959       // these cases in modTInfo, so we can evaluate those regions as part of
1960       // the region chain in META
1961       //if (hbi.foundRegionDir == null) continue;
1962       //if (hbi.deployedOn.size() != 1) continue;
1963       if (hbi.deployedOn.size() == 0) continue;
1964 
1965       // We should be safe here
1966       TableName tableName = hbi.metaEntry.getTable();
1967       TableInfo modTInfo = tablesInfo.get(tableName);
1968       if (modTInfo == null) {
1969         modTInfo = new TableInfo(tableName);
1970       }
1971       for (ServerName server : hbi.deployedOn) {
1972         modTInfo.addServer(server);
1973       }
1974 
1975       if (!hbi.isSkipChecks()) {
1976         modTInfo.addRegionInfo(hbi);
1977       }
1978 
1979       tablesInfo.put(tableName, modTInfo);
1980     }
1981 
1982     loadTableInfosForTablesWithNoRegion();
1983 
1984     for (TableInfo tInfo : tablesInfo.values()) {
1985       TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
1986       if (!tInfo.checkRegionChain(handler)) {
1987         errors.report("Found inconsistency in table " + tInfo.getName());
1988       }
1989     }
1990     return tablesInfo;
1991   }
1992 
1993   /** Loads table info's for tables that may not have been included, since there are no
1994    * regions reported for the table, but table dir is there in hdfs
1995    */
1996   private void loadTableInfosForTablesWithNoRegion() throws IOException {
1997     Map<String, HTableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll();
1998     for (HTableDescriptor htd : allTables.values()) {
1999       if (checkMetaOnly && !htd.isMetaTable()) {
2000         continue;
2001       }
2002 
2003       TableName tableName = htd.getTableName();
2004       if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) {
2005         TableInfo tableInfo = new TableInfo(tableName);
2006         tableInfo.htds.add(htd);
2007         tablesInfo.put(htd.getTableName(), tableInfo);
2008       }
2009     }
2010   }
2011 
2012   /**
2013    * Merge hdfs data by moving from contained HbckInfo into targetRegionDir.
2014    * @return number of file move fixes done to merge regions.
2015    */
2016   public int mergeRegionDirs(Path targetRegionDir, HbckInfo contained) throws IOException {
2017     int fileMoves = 0;
2018     String thread = Thread.currentThread().getName();
2019     LOG.debug("[" + thread + "] Contained region dir after close and pause");
2020     debugLsr(contained.getHdfsRegionDir());
2021 
2022     // rename the contained into the container.
2023     FileSystem fs = targetRegionDir.getFileSystem(getConf());
2024     FileStatus[] dirs = null;
2025     try {
2026       dirs = fs.listStatus(contained.getHdfsRegionDir());
2027     } catch (FileNotFoundException fnfe) {
2028       // region we are attempting to merge in is not present!  Since this is a merge, there is
2029       // no harm skipping this region if it does not exist.
2030       if (!fs.exists(contained.getHdfsRegionDir())) {
2031         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2032             + " is missing. Assuming already sidelined or moved.");
2033       } else {
2034         sidelineRegionDir(fs, contained);
2035       }
2036       return fileMoves;
2037     }
2038 
2039     if (dirs == null) {
2040       if (!fs.exists(contained.getHdfsRegionDir())) {
2041         LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()
2042             + " already sidelined.");
2043       } else {
2044         sidelineRegionDir(fs, contained);
2045       }
2046       return fileMoves;
2047     }
2048 
2049     for (FileStatus cf : dirs) {
2050       Path src = cf.getPath();
2051       Path dst =  new Path(targetRegionDir, src.getName());
2052 
2053       if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {
2054         // do not copy the old .regioninfo file.
2055         continue;
2056       }
2057 
2058       if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {
2059         // do not copy the .oldlogs files
2060         continue;
2061       }
2062 
2063       LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);
2064       // FileSystem.rename is inconsistent with directories -- if the
2065       // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
2066       // it moves the src into the dst dir resulting in (foo/a/b).  If
2067       // the dst does not exist, and the src a dir, src becomes dst. (foo/b)
2068       for (FileStatus hfile : fs.listStatus(src)) {
2069         boolean success = fs.rename(hfile.getPath(), dst);
2070         if (success) {
2071           fileMoves++;
2072         }
2073       }
2074       LOG.debug("[" + thread + "] Sideline directory contents:");
2075       debugLsr(targetRegionDir);
2076     }
2077 
2078     // if all success.
2079     sidelineRegionDir(fs, contained);
2080     LOG.info("[" + thread + "] Sidelined region dir "+ contained.getHdfsRegionDir() + " into " +
2081         getSidelineDir());
2082     debugLsr(contained.getHdfsRegionDir());
2083 
2084     return fileMoves;
2085   }
2086 
2087 
2088   static class WorkItemOverlapMerge implements Callable<Void> {
2089     private TableIntegrityErrorHandler handler;
2090     Collection<HbckInfo> overlapgroup;
2091 
2092     WorkItemOverlapMerge(Collection<HbckInfo> overlapgroup, TableIntegrityErrorHandler handler) {
2093       this.handler = handler;
2094       this.overlapgroup = overlapgroup;
2095     }
2096 
2097     @Override
2098     public Void call() throws Exception {
2099       handler.handleOverlapGroup(overlapgroup);
2100       return null;
2101     }
2102   };
2103 
2104 
2105   /**
2106    * Maintain information about a particular table.
2107    */
2108   public class TableInfo {
2109     TableName tableName;
2110     TreeSet <ServerName> deployedOn;
2111 
2112     // backwards regions
2113     final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
2114 
2115     // sidelined big overlapped regions
2116     final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
2117 
2118     // region split calculator
2119     final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
2120 
2121     // Histogram of different HTableDescriptors found.  Ideally there is only one!
2122     final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
2123 
2124     // key = start split, values = set of splits in problem group
2125     final Multimap<byte[], HbckInfo> overlapGroups =
2126       TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp);
2127 
2128     TableInfo(TableName name) {
2129       this.tableName = name;
2130       deployedOn = new TreeSet <ServerName>();
2131     }
2132 
2133     /**
2134      * @return descriptor common to all regions.  null if are none or multiple!
2135      */
2136     private HTableDescriptor getHTD() {
2137       if (htds.size() == 1) {
2138         return (HTableDescriptor)htds.toArray()[0];
2139       } else {
2140         LOG.error("None/Multiple table descriptors found for table '"
2141           + tableName + "' regions: " + htds);
2142       }
2143       return null;
2144     }
2145 
2146     public void addRegionInfo(HbckInfo hir) {
2147       if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
2148         // end key is absolute end key, just add it.
2149         sc.add(hir);
2150         return;
2151       }
2152 
2153       // if not the absolute end key, check for cycle
2154       if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
2155         errors.reportError(
2156             ERROR_CODE.REGION_CYCLE,
2157             String.format("The endkey for this region comes before the "
2158                 + "startkey, startkey=%s, endkey=%s",
2159                 Bytes.toStringBinary(hir.getStartKey()),
2160                 Bytes.toStringBinary(hir.getEndKey())), this, hir);
2161         backwards.add(hir);
2162         return;
2163       }
2164 
2165       // main case, add to split calculator
2166       sc.add(hir);
2167     }
2168 
2169     public void addServer(ServerName server) {
2170       this.deployedOn.add(server);
2171     }
2172 
2173     public TableName getName() {
2174       return tableName;
2175     }
2176 
2177     public int getNumRegions() {
2178       return sc.getStarts().size() + backwards.size();
2179     }
2180 
2181     private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl {
2182       ErrorReporter errors;
2183 
2184       IntegrityFixSuggester(TableInfo ti, ErrorReporter errors) {
2185         this.errors = errors;
2186         setTableInfo(ti);
2187       }
2188 
2189       @Override
2190       public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{
2191         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2192             "First region should start with an empty key.  You need to "
2193             + " create a new region and regioninfo in HDFS to plug the hole.",
2194             getTableInfo(), hi);
2195       }
2196 
2197       @Override
2198       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2199         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2200             "Last region should end with an empty key. You need to "
2201                 + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo());
2202       }
2203 
2204       @Override
2205       public void handleDegenerateRegion(HbckInfo hi) throws IOException{
2206         errors.reportError(ERROR_CODE.DEGENERATE_REGION,
2207             "Region has the same start and end key.", getTableInfo(), hi);
2208       }
2209 
2210       @Override
2211       public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{
2212         byte[] key = r1.getStartKey();
2213         // dup start key
2214         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2215             "Multiple regions have the same startkey: "
2216             + Bytes.toStringBinary(key), getTableInfo(), r1);
2217         errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
2218             "Multiple regions have the same startkey: "
2219             + Bytes.toStringBinary(key), getTableInfo(), r2);
2220       }
2221 
2222       @Override
2223       public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{
2224         errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
2225             "There is an overlap in the region chain.",
2226             getTableInfo(), hi1, hi2);
2227       }
2228 
2229       @Override
2230       public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{
2231         errors.reportError(
2232             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2233             "There is a hole in the region chain between "
2234                 + Bytes.toStringBinary(holeStart) + " and "
2235                 + Bytes.toStringBinary(holeStop)
2236                 + ".  You need to create a new .regioninfo and region "
2237                 + "dir in hdfs to plug the hole.");
2238       }
2239     };
2240 
2241     /**
2242      * This handler fixes integrity errors from hdfs information.  There are
2243      * basically three classes of integrity problems 1) holes, 2) overlaps, and
2244      * 3) invalid regions.
2245      *
2246      * This class overrides methods that fix holes and the overlap group case.
2247      * Individual cases of particular overlaps are handled by the general
2248      * overlap group merge repair case.
2249      *
2250      * If hbase is online, this forces regions offline before doing merge
2251      * operations.
2252      */
2253     private class HDFSIntegrityFixer extends IntegrityFixSuggester {
2254       Configuration conf;
2255 
2256       boolean fixOverlaps = true;
2257 
2258       HDFSIntegrityFixer(TableInfo ti, ErrorReporter errors, Configuration conf,
2259           boolean fixHoles, boolean fixOverlaps) {
2260         super(ti, errors);
2261         this.conf = conf;
2262         this.fixOverlaps = fixOverlaps;
2263         // TODO properly use fixHoles
2264       }
2265 
2266       /**
2267        * This is a special case hole -- when the first region of a table is
2268        * missing from META, HBase doesn't acknowledge the existance of the
2269        * table.
2270        */
2271       @Override
2272       public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException {
2273         errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
2274             "First region should start with an empty key.  Creating a new " +
2275             "region and regioninfo in HDFS to plug the hole.",
2276             getTableInfo(), next);
2277         HTableDescriptor htd = getTableInfo().getHTD();
2278         // from special EMPTY_START_ROW to next region's startKey
2279         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(),
2280             HConstants.EMPTY_START_ROW, next.getStartKey());
2281 
2282         // TODO test
2283         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2284         LOG.info("Table region start key was not empty.  Created new empty region: "
2285             + newRegion + " " +region);
2286         fixes++;
2287       }
2288 
2289       @Override
2290       public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
2291         errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
2292             "Last region should end with an empty key.  Creating a new "
2293                 + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
2294         HTableDescriptor htd = getTableInfo().getHTD();
2295         // from curEndKey to EMPTY_START_ROW
2296         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey,
2297             HConstants.EMPTY_START_ROW);
2298 
2299         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2300         LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
2301             + " " + region);
2302         fixes++;
2303       }
2304 
2305       /**
2306        * There is a hole in the hdfs regions that violates the table integrity
2307        * rules.  Create a new empty region that patches the hole.
2308        */
2309       @Override
2310       public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException {
2311         errors.reportError(
2312             ERROR_CODE.HOLE_IN_REGION_CHAIN,
2313             "There is a hole in the region chain between "
2314                 + Bytes.toStringBinary(holeStartKey) + " and "
2315                 + Bytes.toStringBinary(holeStopKey)
2316                 + ".  Creating a new regioninfo and region "
2317                 + "dir in hdfs to plug the hole.");
2318         HTableDescriptor htd = getTableInfo().getHTD();
2319         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey);
2320         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2321         LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region);
2322         fixes++;
2323       }
2324 
2325       /**
2326        * This takes set of overlapping regions and merges them into a single
2327        * region.  This covers cases like degenerate regions, shared start key,
2328        * general overlaps, duplicate ranges, and partial overlapping regions.
2329        *
2330        * Cases:
2331        * - Clean regions that overlap
2332        * - Only .oldlogs regions (can't find start/stop range, or figure out)
2333        *
2334        * This is basically threadsafe, except for the fixer increment in mergeOverlaps.
2335        */
2336       @Override
2337       public void handleOverlapGroup(Collection<HbckInfo> overlap)
2338           throws IOException {
2339         Preconditions.checkNotNull(overlap);
2340         Preconditions.checkArgument(overlap.size() >0);
2341 
2342         if (!this.fixOverlaps) {
2343           LOG.warn("Not attempting to repair overlaps.");
2344           return;
2345         }
2346 
2347         if (overlap.size() > maxMerge) {
2348           LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
2349             "regions which is greater than " + maxMerge + ", the max number of regions to merge");
2350           if (sidelineBigOverlaps) {
2351             // we only sideline big overlapped groups that exceeds the max number of regions to merge
2352             sidelineBigOverlaps(overlap);
2353           }
2354           return;
2355         }
2356 
2357         mergeOverlaps(overlap);
2358       }
2359 
2360       void mergeOverlaps(Collection<HbckInfo> overlap)
2361           throws IOException {
2362         String thread = Thread.currentThread().getName();
2363         LOG.info("== [" + thread + "] Merging regions into one region: "
2364           + Joiner.on(",").join(overlap));
2365         // get the min / max range and close all concerned regions
2366         Pair<byte[], byte[]> range = null;
2367         for (HbckInfo hi : overlap) {
2368           if (range == null) {
2369             range = new Pair<byte[], byte[]>(hi.getStartKey(), hi.getEndKey());
2370           } else {
2371             if (RegionSplitCalculator.BYTES_COMPARATOR
2372                 .compare(hi.getStartKey(), range.getFirst()) < 0) {
2373               range.setFirst(hi.getStartKey());
2374             }
2375             if (RegionSplitCalculator.BYTES_COMPARATOR
2376                 .compare(hi.getEndKey(), range.getSecond()) > 0) {
2377               range.setSecond(hi.getEndKey());
2378             }
2379           }
2380           // need to close files so delete can happen.
2381           LOG.debug("[" + thread + "] Closing region before moving data around: " +  hi);
2382           LOG.debug("[" + thread + "] Contained region dir before close");
2383           debugLsr(hi.getHdfsRegionDir());
2384           try {
2385             LOG.info("[" + thread + "] Closing region: " + hi);
2386             closeRegion(hi);
2387           } catch (IOException ioe) {
2388             LOG.warn("[" + thread + "] Was unable to close region " + hi
2389               + ".  Just continuing... ", ioe);
2390           } catch (InterruptedException e) {
2391             LOG.warn("[" + thread + "] Was unable to close region " + hi
2392               + ".  Just continuing... ", e);
2393           }
2394 
2395           try {
2396             LOG.info("[" + thread + "] Offlining region: " + hi);
2397             offline(hi.getRegionName());
2398           } catch (IOException ioe) {
2399             LOG.warn("[" + thread + "] Unable to offline region from master: " + hi
2400               + ".  Just continuing... ", ioe);
2401           }
2402         }
2403 
2404         // create new empty container region.
2405         HTableDescriptor htd = getTableInfo().getHTD();
2406         // from start key to end Key
2407         HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(),
2408             range.getSecond());
2409         HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
2410         LOG.info("[" + thread + "] Created new empty container region: " +
2411             newRegion + " to contain regions: " + Joiner.on(",").join(overlap));
2412         debugLsr(region.getRegionFileSystem().getRegionDir());
2413 
2414         // all target regions are closed, should be able to safely cleanup.
2415         boolean didFix= false;
2416         Path target = region.getRegionFileSystem().getRegionDir();
2417         for (HbckInfo contained : overlap) {
2418           LOG.info("[" + thread + "] Merging " + contained  + " into " + target );
2419           int merges = mergeRegionDirs(target, contained);
2420           if (merges > 0) {
2421             didFix = true;
2422           }
2423         }
2424         if (didFix) {
2425           fixes++;
2426         }
2427       }
2428 
2429       /**
2430        * Sideline some regions in a big overlap group so that it
2431        * will have fewer regions, and it is easier to merge them later on.
2432        *
2433        * @param bigOverlap the overlapped group with regions more than maxMerge
2434        * @throws IOException
2435        */
2436       void sidelineBigOverlaps(
2437           Collection<HbckInfo> bigOverlap) throws IOException {
2438         int overlapsToSideline = bigOverlap.size() - maxMerge;
2439         if (overlapsToSideline > maxOverlapsToSideline) {
2440           overlapsToSideline = maxOverlapsToSideline;
2441         }
2442         List<HbckInfo> regionsToSideline =
2443           RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
2444         FileSystem fs = FileSystem.get(conf);
2445         for (HbckInfo regionToSideline: regionsToSideline) {
2446           try {
2447             LOG.info("Closing region: " + regionToSideline);
2448             closeRegion(regionToSideline);
2449           } catch (IOException ioe) {
2450             LOG.warn("Was unable to close region " + regionToSideline
2451               + ".  Just continuing... ", ioe);
2452           } catch (InterruptedException e) {
2453             LOG.warn("Was unable to close region " + regionToSideline
2454               + ".  Just continuing... ", e);
2455           }
2456 
2457           try {
2458             LOG.info("Offlining region: " + regionToSideline);
2459             offline(regionToSideline.getRegionName());
2460           } catch (IOException ioe) {
2461             LOG.warn("Unable to offline region from master: " + regionToSideline
2462               + ".  Just continuing... ", ioe);
2463           }
2464 
2465           LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
2466           Path sidelineRegionDir = sidelineRegionDir(fs, TO_BE_LOADED, regionToSideline);
2467           if (sidelineRegionDir != null) {
2468             sidelinedRegions.put(sidelineRegionDir, regionToSideline);
2469             LOG.info("After sidelined big overlapped region: "
2470               + regionToSideline.getRegionNameAsString()
2471               + " to " + sidelineRegionDir.toString());
2472             fixes++;
2473           }
2474         }
2475       }
2476     }
2477 
2478     /**
2479      * Check the region chain (from META) of this table.  We are looking for
2480      * holes, overlaps, and cycles.
2481      * @return false if there are errors
2482      * @throws IOException
2483      */
2484     public boolean checkRegionChain(TableIntegrityErrorHandler handler) throws IOException {
2485       // When table is disabled no need to check for the region chain. Some of the regions
2486       // accidently if deployed, this below code might report some issues like missing start
2487       // or end regions or region hole in chain and may try to fix which is unwanted.
2488       if (disabledTables.contains(this.tableName)) {
2489         return true;
2490       }
2491       int originalErrorsCount = errors.getErrorList().size();
2492       Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
2493       SortedSet<byte[]> splits = sc.getSplits();
2494 
2495       byte[] prevKey = null;
2496       byte[] problemKey = null;
2497 
2498       if (splits.size() == 0) {
2499         // no region for this table
2500         handler.handleHoleInRegionChain(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
2501       }
2502 
2503       for (byte[] key : splits) {
2504         Collection<HbckInfo> ranges = regions.get(key);
2505         if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
2506           for (HbckInfo rng : ranges) {
2507             handler.handleRegionStartKeyNotEmpty(rng);
2508           }
2509         }
2510 
2511         // check for degenerate ranges
2512         for (HbckInfo rng : ranges) {
2513           // special endkey case converts '' to null
2514           byte[] endKey = rng.getEndKey();
2515           endKey = (endKey.length == 0) ? null : endKey;
2516           if (Bytes.equals(rng.getStartKey(),endKey)) {
2517             handler.handleDegenerateRegion(rng);
2518           }
2519         }
2520 
2521         if (ranges.size() == 1) {
2522           // this split key is ok -- no overlap, not a hole.
2523           if (problemKey != null) {
2524             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2525           }
2526           problemKey = null; // fell through, no more problem.
2527         } else if (ranges.size() > 1) {
2528           // set the new problem key group name, if already have problem key, just
2529           // keep using it.
2530           if (problemKey == null) {
2531             // only for overlap regions.
2532             LOG.warn("Naming new problem group: " + Bytes.toStringBinary(key));
2533             problemKey = key;
2534           }
2535           overlapGroups.putAll(problemKey, ranges);
2536 
2537           // record errors
2538           ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
2539           //  this dumb and n^2 but this shouldn't happen often
2540           for (HbckInfo r1 : ranges) {
2541             subRange.remove(r1);
2542             for (HbckInfo r2 : subRange) {
2543               if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
2544                 handler.handleDuplicateStartKeys(r1,r2);
2545               } else {
2546                 // overlap
2547                 handler.handleOverlapInRegionChain(r1, r2);
2548               }
2549             }
2550           }
2551 
2552         } else if (ranges.size() == 0) {
2553           if (problemKey != null) {
2554             LOG.warn("reached end of problem group: " + Bytes.toStringBinary(key));
2555           }
2556           problemKey = null;
2557 
2558           byte[] holeStopKey = sc.getSplits().higher(key);
2559           // if higher key is null we reached the top.
2560           if (holeStopKey != null) {
2561             // hole
2562             handler.handleHoleInRegionChain(key, holeStopKey);
2563           }
2564         }
2565         prevKey = key;
2566       }
2567 
2568       // When the last region of a table is proper and having an empty end key, 'prevKey'
2569       // will be null.
2570       if (prevKey != null) {
2571         handler.handleRegionEndKeyNotEmpty(prevKey);
2572       }
2573 
2574       // TODO fold this into the TableIntegrityHandler
2575       if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {
2576         LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" +
2577             " false to run serially.");
2578         boolean ok = handleOverlapsParallel(handler, prevKey);
2579         if (!ok) {
2580           return false;
2581         }
2582       } else {
2583         LOG.info("Handling overlap merges serially.  set hbasefsck.overlap.merge.parallel to" +
2584             " true to run in parallel.");
2585         for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2586           handler.handleOverlapGroup(overlap);
2587         }
2588       }
2589 
2590       if (details) {
2591         // do full region split map dump
2592         errors.print("---- Table '"  +  this.tableName
2593             + "': region split map");
2594         dump(splits, regions);
2595         errors.print("---- Table '"  +  this.tableName
2596             + "': overlap groups");
2597         dumpOverlapProblems(overlapGroups);
2598         errors.print("There are " + overlapGroups.keySet().size()
2599             + " overlap groups with " + overlapGroups.size()
2600             + " overlapping regions");
2601       }
2602       if (!sidelinedRegions.isEmpty()) {
2603         LOG.warn("Sidelined big overlapped regions, please bulk load them!");
2604         errors.print("---- Table '"  +  this.tableName
2605             + "': sidelined big overlapped regions");
2606         dumpSidelinedRegions(sidelinedRegions);
2607       }
2608       return errors.getErrorList().size() == originalErrorsCount;
2609     }
2610 
2611     private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey)
2612         throws IOException {
2613       // we parallelize overlap handler for the case we have lots of groups to fix.  We can
2614       // safely assume each group is independent.
2615       List<WorkItemOverlapMerge> merges = new ArrayList<WorkItemOverlapMerge>(overlapGroups.size());
2616       List<Future<Void>> rets;
2617       for (Collection<HbckInfo> overlap : overlapGroups.asMap().values()) {
2618         //
2619         merges.add(new WorkItemOverlapMerge(overlap, handler));
2620       }
2621       try {
2622         rets = executor.invokeAll(merges);
2623       } catch (InterruptedException e) {
2624         LOG.error("Overlap merges were interrupted", e);
2625         return false;
2626       }
2627       for(int i=0; i<merges.size(); i++) {
2628         WorkItemOverlapMerge work = merges.get(i);
2629         Future<Void> f = rets.get(i);
2630         try {
2631           f.get();
2632         } catch(ExecutionException e) {
2633           LOG.warn("Failed to merge overlap group" + work, e.getCause());
2634         } catch (InterruptedException e) {
2635           LOG.error("Waiting for overlap merges was interrupted", e);
2636           return false;
2637         }
2638       }
2639       return true;
2640     }
2641 
2642     /**
2643      * This dumps data in a visually reasonable way for visual debugging
2644      *
2645      * @param splits
2646      * @param regions
2647      */
2648     void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) {
2649       // we display this way because the last end key should be displayed as well.
2650       StringBuilder sb = new StringBuilder();
2651       for (byte[] k : splits) {
2652         sb.setLength(0); // clear out existing buffer, if any.
2653         sb.append(Bytes.toStringBinary(k) + ":\t");
2654         for (HbckInfo r : regions.get(k)) {
2655           sb.append("[ "+ r.toString() + ", "
2656               + Bytes.toStringBinary(r.getEndKey())+ "]\t");
2657         }
2658         errors.print(sb.toString());
2659       }
2660     }
2661   }
2662 
2663   public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) {
2664     // we display this way because the last end key should be displayed as
2665     // well.
2666     for (byte[] k : regions.keySet()) {
2667       errors.print(Bytes.toStringBinary(k) + ":");
2668       for (HbckInfo r : regions.get(k)) {
2669         errors.print("[ " + r.toString() + ", "
2670             + Bytes.toStringBinary(r.getEndKey()) + "]");
2671       }
2672       errors.print("----");
2673     }
2674   }
2675 
2676   public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
2677     for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) {
2678       TableName tableName = entry.getValue().getTableName();
2679       Path path = entry.getKey();
2680       errors.print("This sidelined region dir should be bulk loaded: "
2681         + path.toString());
2682       errors.print("Bulk load command looks like: "
2683         + "hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles "
2684         + path.toUri().getPath() + " "+ tableName);
2685     }
2686   }
2687 
2688   public Multimap<byte[], HbckInfo> getOverlapGroups(
2689       TableName table) {
2690     TableInfo ti = tablesInfo.get(table);
2691     return ti.overlapGroups;
2692   }
2693 
2694   /**
2695    * Return a list of user-space table names whose metadata have not been
2696    * modified in the last few milliseconds specified by timelag
2697    * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
2698    * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
2699    * milliseconds specified by timelag, then the table is a candidate to be returned.
2700    * @return tables that have not been modified recently
2701    * @throws IOException if an error is encountered
2702    */
2703   HTableDescriptor[] getTables(AtomicInteger numSkipped) {
2704     List<TableName> tableNames = new ArrayList<TableName>();
2705     long now = System.currentTimeMillis();
2706 
2707     for (HbckInfo hbi : regionInfoMap.values()) {
2708       MetaEntry info = hbi.metaEntry;
2709 
2710       // if the start key is zero, then we have found the first region of a table.
2711       // pick only those tables that were not modified in the last few milliseconds.
2712       if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
2713         if (info.modTime + timelag < now) {
2714           tableNames.add(info.getTable());
2715         } else {
2716           numSkipped.incrementAndGet(); // one more in-flux table
2717         }
2718       }
2719     }
2720     return getHTableDescriptors(tableNames);
2721   }
2722 
2723   HTableDescriptor[] getHTableDescriptors(List<TableName> tableNames) {
2724     HTableDescriptor[] htd = new HTableDescriptor[0];
2725     HBaseAdmin admin = null;
2726     try {
2727       LOG.info("getHTableDescriptors == tableNames => " + tableNames);
2728       admin = new HBaseAdmin(getConf());
2729       htd = admin.getTableDescriptorsByTableName(tableNames);
2730     } catch (IOException e) {
2731       LOG.debug("Exception getting table descriptors", e);
2732     } finally {
2733       if (admin != null) {
2734         try {
2735           admin.close();
2736         } catch (IOException e) {
2737           LOG.debug("Exception closing HBaseAdmin", e);
2738         }
2739       }
2740     }
2741     return htd;
2742   }
2743 
2744   /**
2745    * Gets the entry in regionInfo corresponding to the the given encoded
2746    * region name. If the region has not been seen yet, a new entry is added
2747    * and returned.
2748    */
2749   private synchronized HbckInfo getOrCreateInfo(String name) {
2750     HbckInfo hbi = regionInfoMap.get(name);
2751     if (hbi == null) {
2752       hbi = new HbckInfo(null);
2753       regionInfoMap.put(name, hbi);
2754     }
2755     return hbi;
2756   }
2757 
2758   private void checkAndFixTableLocks() throws IOException {
2759     TableLockChecker checker = new TableLockChecker(createZooKeeperWatcher(), errors);
2760     checker.checkTableLocks();
2761 
2762     if (this.fixTableLocks) {
2763       checker.fixExpiredTableLocks();
2764     }
2765   }
2766 
2767   /**
2768     * Check values in regionInfo for hbase:meta
2769     * Check if zero or more than one regions with hbase:meta are found.
2770     * If there are inconsistencies (i.e. zero or more than one regions
2771     * pretend to be holding the hbase:meta) try to fix that and report an error.
2772     * @throws IOException from HBaseFsckRepair functions
2773    * @throws KeeperException
2774    * @throws InterruptedException
2775     */
2776   boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {
2777     List<HbckInfo> metaRegions = Lists.newArrayList();
2778     for (HbckInfo value : regionInfoMap.values()) {
2779       if (value.metaEntry != null && value.metaEntry.isMetaRegion()) {
2780         metaRegions.add(value);
2781       }
2782     }
2783 
2784     // There will be always one entry in regionInfoMap corresponding to hbase:meta
2785     // Check the deployed servers. It should be exactly one server.
2786     List<ServerName> servers = new ArrayList<ServerName>();
2787     HbckInfo metaHbckInfo = null;
2788     if (!metaRegions.isEmpty()) {
2789       metaHbckInfo = metaRegions.get(0);
2790       servers = metaHbckInfo.deployedOn;
2791     }
2792     if (servers.size() != 1) {
2793       if (servers.size() == 0) {
2794         errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta is not found on any region.");
2795         if (shouldFixAssignments()) {
2796           errors.print("Trying to fix a problem with hbase:meta..");
2797           setShouldRerun();
2798           // try to fix it (treat it as unassigned region)
2799           HBaseFsckRepair.fixUnassigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
2800           HBaseFsckRepair.waitUntilAssigned(admin, HRegionInfo.FIRST_META_REGIONINFO);
2801         }
2802       } else if (servers.size() > 1) {
2803         errors
2804             .reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta is found on more than one region.");
2805         if (shouldFixAssignments()) {
2806           if (metaHbckInfo == null) {
2807             errors.print(
2808               "Unable to fix problem with hbase:meta due to hbase:meta region info missing");
2809             return false;
2810           }
2811           errors.print("Trying to fix a problem with hbase:meta..");
2812           setShouldRerun();
2813           // try fix it (treat is a dupe assignment)
2814           HBaseFsckRepair.fixMultiAssignment(admin, metaHbckInfo.metaEntry, servers);
2815         }
2816       }
2817       // rerun hbck with hopefully fixed META
2818       return false;
2819     }
2820     // no errors, so continue normally
2821     return true;
2822   }
2823 
2824   /**
2825    * Scan hbase:meta, adding all regions found to the regionInfo map.
2826    * @throws IOException if an error is encountered
2827    */
2828   boolean loadMetaEntries() throws IOException {
2829     MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
2830       int countRecord = 1;
2831 
2832       // comparator to sort KeyValues with latest modtime
2833       final Comparator<Cell> comp = new Comparator<Cell>() {
2834         @Override
2835         public int compare(Cell k1, Cell k2) {
2836           return (int)(k1.getTimestamp() - k2.getTimestamp());
2837         }
2838       };
2839 
2840       @Override
2841       public boolean processRow(Result result) throws IOException {
2842         try {
2843 
2844           // record the latest modification of this META record
2845           long ts =  Collections.max(result.listCells(), comp).getTimestamp();
2846           Pair<HRegionInfo, ServerName> pair = HRegionInfo.getHRegionInfoAndServerName(result);
2847           if (pair == null || pair.getFirst() == null) {
2848             emptyRegionInfoQualifiers.add(result);
2849             errors.reportError(ERROR_CODE.EMPTY_META_CELL,
2850               "Empty REGIONINFO_QUALIFIER found in hbase:meta");
2851             return true;
2852           }
2853           ServerName sn = null;
2854           if (pair.getSecond() != null) {
2855             sn = pair.getSecond();
2856           }
2857           HRegionInfo hri = pair.getFirst();
2858           if (!(isTableIncluded(hri.getTable())
2859               || hri.isMetaRegion())) {
2860             return true;
2861           }
2862           PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(result);
2863           MetaEntry m = new MetaEntry(hri, sn, ts, daughters.getFirst(), daughters.getSecond());
2864           HbckInfo previous = regionInfoMap.get(hri.getEncodedName());
2865           if (previous == null) {
2866             regionInfoMap.put(hri.getEncodedName(), new HbckInfo(m));
2867           } else if (previous.metaEntry == null) {
2868             previous.metaEntry = m;
2869           } else {
2870             throw new IOException("Two entries in hbase:meta are same " + previous);
2871           }
2872 
2873           PairOfSameType<HRegionInfo> mergeRegions = HRegionInfo.getMergeRegions(result);
2874           for (HRegionInfo mergeRegion : new HRegionInfo[] {
2875               mergeRegions.getFirst(), mergeRegions.getSecond() }) {
2876             if (mergeRegion != null) {
2877               // This region is already been merged
2878               HbckInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());
2879               hbInfo.setMerged(true);
2880             }
2881           }
2882 
2883           // show proof of progress to the user, once for every 100 records.
2884           if (countRecord % 100 == 0) {
2885             errors.progress();
2886           }
2887           countRecord++;
2888           return true;
2889         } catch (RuntimeException e) {
2890           LOG.error("Result=" + result);
2891           throw e;
2892         }
2893       }
2894     };
2895     if (!checkMetaOnly) {
2896       // Scan hbase:meta to pick up user regions
2897       MetaScanner.metaScan(getConf(), visitor);
2898     }
2899 
2900     errors.print("");
2901     return true;
2902   }
2903 
2904   /**
2905    * Stores the regioninfo entries scanned from META
2906    */
2907   static class MetaEntry extends HRegionInfo {
2908     ServerName regionServer;   // server hosting this region
2909     long modTime;          // timestamp of most recent modification metadata
2910     HRegionInfo splitA, splitB; //split daughters
2911 
2912     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime) {
2913       this(rinfo, regionServer, modTime, null, null);
2914     }
2915 
2916     public MetaEntry(HRegionInfo rinfo, ServerName regionServer, long modTime,
2917         HRegionInfo splitA, HRegionInfo splitB) {
2918       super(rinfo);
2919       this.regionServer = regionServer;
2920       this.modTime = modTime;
2921       this.splitA = splitA;
2922       this.splitB = splitB;
2923     }
2924 
2925     @Override
2926     public boolean equals(Object o) {
2927       boolean superEq = super.equals(o);
2928       if (!superEq) {
2929         return superEq;
2930       }
2931 
2932       MetaEntry me = (MetaEntry) o;
2933       if (!regionServer.equals(me.regionServer)) {
2934         return false;
2935       }
2936       return (modTime == me.modTime);
2937     }
2938 
2939     @Override
2940     public int hashCode() {
2941       int hash = Arrays.hashCode(getRegionName());
2942       hash ^= getRegionId();
2943       hash ^= Arrays.hashCode(getStartKey());
2944       hash ^= Arrays.hashCode(getEndKey());
2945       hash ^= Boolean.valueOf(isOffline()).hashCode();
2946       hash ^= getTable().hashCode();
2947       if (regionServer != null) {
2948         hash ^= regionServer.hashCode();
2949       }
2950       hash ^= modTime;
2951       return hash;
2952     }
2953   }
2954 
2955   /**
2956    * Stores the regioninfo entries from HDFS
2957    */
2958   static class HdfsEntry {
2959     HRegionInfo hri;
2960     Path hdfsRegionDir = null;
2961     long hdfsRegionDirModTime  = 0;
2962     boolean hdfsRegioninfoFilePresent = false;
2963     boolean hdfsOnlyEdits = false;
2964   }
2965 
2966   /**
2967    * Stores the regioninfo retrieved from Online region servers.
2968    */
2969   static class OnlineEntry {
2970     HRegionInfo hri;
2971     ServerName hsa;
2972 
2973     @Override
2974     public String toString() {
2975       return hsa.toString() + ";" + hri.getRegionNameAsString();
2976     }
2977   }
2978 
2979   /**
2980    * Maintain information about a particular region.  It gathers information
2981    * from three places -- HDFS, META, and region servers.
2982    */
2983   public static class HbckInfo implements KeyRange {
2984     private MetaEntry metaEntry = null; // info in META
2985     private HdfsEntry hdfsEntry = null; // info in HDFS
2986     private List<OnlineEntry> deployedEntries = Lists.newArrayList(); // on Region Server
2987     private List<ServerName> deployedOn = Lists.newArrayList(); // info on RS's
2988     private boolean skipChecks = false; // whether to skip further checks to this region info.
2989     private boolean isMerged = false;// whether this region has already been merged into another one
2990 
2991     HbckInfo(MetaEntry metaEntry) {
2992       this.metaEntry = metaEntry;
2993     }
2994 
2995     public synchronized void addServer(HRegionInfo hri, ServerName server) {
2996       OnlineEntry rse = new OnlineEntry() ;
2997       rse.hri = hri;
2998       rse.hsa = server;
2999       this.deployedEntries.add(rse);
3000       this.deployedOn.add(server);
3001     }
3002 
3003     @Override
3004     public synchronized String toString() {
3005       StringBuilder sb = new StringBuilder();
3006       sb.append("{ meta => ");
3007       sb.append((metaEntry != null)? metaEntry.getRegionNameAsString() : "null");
3008       sb.append( ", hdfs => " + getHdfsRegionDir());
3009       sb.append( ", deployed => " + Joiner.on(", ").join(deployedEntries));
3010       sb.append(" }");
3011       return sb.toString();
3012     }
3013 
3014     @Override
3015     public byte[] getStartKey() {
3016       if (this.metaEntry != null) {
3017         return this.metaEntry.getStartKey();
3018       } else if (this.hdfsEntry != null) {
3019         return this.hdfsEntry.hri.getStartKey();
3020       } else {
3021         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3022         return null;
3023       }
3024     }
3025 
3026     @Override
3027     public byte[] getEndKey() {
3028       if (this.metaEntry != null) {
3029         return this.metaEntry.getEndKey();
3030       } else if (this.hdfsEntry != null) {
3031         return this.hdfsEntry.hri.getEndKey();
3032       } else {
3033         LOG.error("Entry " + this + " has no meta or hdfs region start key.");
3034         return null;
3035       }
3036     }
3037 
3038     public TableName getTableName() {
3039       if (this.metaEntry != null) {
3040         return this.metaEntry.getTable();
3041       } else if (this.hdfsEntry != null) {
3042         // we are only guaranteed to have a path and not an HRI for hdfsEntry,
3043         // so we get the name from the Path
3044         Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent();
3045         return FSUtils.getTableName(tableDir);
3046       } else {
3047         // Currently no code exercises this path, but we could add one for
3048         // getting table name from OnlineEntry
3049         return null;
3050       }
3051     }
3052 
3053     public String getRegionNameAsString() {
3054       if (metaEntry != null) {
3055         return metaEntry.getRegionNameAsString();
3056       } else if (hdfsEntry != null) {
3057         if (hdfsEntry.hri != null) {
3058           return hdfsEntry.hri.getRegionNameAsString();
3059         }
3060       }
3061       return null;
3062     }
3063 
3064     public byte[] getRegionName() {
3065       if (metaEntry != null) {
3066         return metaEntry.getRegionName();
3067       } else if (hdfsEntry != null) {
3068         return hdfsEntry.hri.getRegionName();
3069       } else {
3070         return null;
3071       }
3072     }
3073 
3074     Path getHdfsRegionDir() {
3075       if (hdfsEntry == null) {
3076         return null;
3077       }
3078       return hdfsEntry.hdfsRegionDir;
3079     }
3080 
3081     boolean containsOnlyHdfsEdits() {
3082       if (hdfsEntry == null) {
3083         return false;
3084       }
3085       return hdfsEntry.hdfsOnlyEdits;
3086     }
3087 
3088     boolean isHdfsRegioninfoPresent() {
3089       if (hdfsEntry == null) {
3090         return false;
3091       }
3092       return hdfsEntry.hdfsRegioninfoFilePresent;
3093     }
3094 
3095     long getModTime() {
3096       if (hdfsEntry == null) {
3097         return 0;
3098       }
3099       return hdfsEntry.hdfsRegionDirModTime;
3100     }
3101 
3102     HRegionInfo getHdfsHRI() {
3103       if (hdfsEntry == null) {
3104         return null;
3105       }
3106       return hdfsEntry.hri;
3107     }
3108 
3109     public void setSkipChecks(boolean skipChecks) {
3110       this.skipChecks = skipChecks;
3111     }
3112 
3113     public boolean isSkipChecks() {
3114       return skipChecks;
3115     }
3116 
3117     public void setMerged(boolean isMerged) {
3118       this.isMerged = isMerged;
3119     }
3120 
3121     public boolean isMerged() {
3122       return this.isMerged;
3123     }
3124   }
3125 
3126   final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
3127     @Override
3128     public int compare(HbckInfo l, HbckInfo r) {
3129       if (l == r) {
3130         // same instance
3131         return 0;
3132       }
3133 
3134       int tableCompare = l.getTableName().compareTo(r.getTableName());
3135       if (tableCompare != 0) {
3136         return tableCompare;
3137       }
3138 
3139       int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3140           l.getStartKey(), r.getStartKey());
3141       if (startComparison != 0) {
3142         return startComparison;
3143       }
3144 
3145       // Special case for absolute endkey
3146       byte[] endKey = r.getEndKey();
3147       endKey = (endKey.length == 0) ? null : endKey;
3148       byte[] endKey2 = l.getEndKey();
3149       endKey2 = (endKey2.length == 0) ? null : endKey2;
3150       int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
3151           endKey2,  endKey);
3152 
3153       if (endComparison != 0) {
3154         return endComparison;
3155       }
3156 
3157       // use regionId as tiebreaker.
3158       // Null is considered after all possible values so make it bigger.
3159       if (l.hdfsEntry == null && r.hdfsEntry == null) {
3160         return 0;
3161       }
3162       if (l.hdfsEntry == null && r.hdfsEntry != null) {
3163         return 1;
3164       }
3165       // l.hdfsEntry must not be null
3166       if (r.hdfsEntry == null) {
3167         return -1;
3168       }
3169       // both l.hdfsEntry and r.hdfsEntry must not be null.
3170       return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
3171     }
3172   };
3173 
3174   /**
3175    * Prints summary of all tables found on the system.
3176    */
3177   private void printTableSummary(SortedMap<TableName, TableInfo> tablesInfo) {
3178     StringBuilder sb = new StringBuilder();
3179     errors.print("Summary:");
3180     for (TableInfo tInfo : tablesInfo.values()) {
3181       if (errors.tableHasErrors(tInfo)) {
3182         errors.print("Table " + tInfo.getName() + " is inconsistent.");
3183       } else {
3184         errors.print("  " + tInfo.getName() + " is okay.");
3185       }
3186       errors.print("    Number of regions: " + tInfo.getNumRegions());
3187       sb.setLength(0); // clear out existing buffer, if any.
3188       sb.append("    Deployed on: ");
3189       for (ServerName server : tInfo.deployedOn) {
3190         sb.append(" " + server.toString());
3191       }
3192       errors.print(sb.toString());
3193     }
3194   }
3195 
3196   static ErrorReporter getErrorReporter(
3197       final Configuration conf) throws ClassNotFoundException {
3198     Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
3199     return (ErrorReporter)ReflectionUtils.newInstance(reporter, conf);
3200   }
3201 
3202   public interface ErrorReporter {
3203     enum ERROR_CODE {
3204       UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
3205       NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED,
3206       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
3207       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
3208       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
3209       ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
3210       WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR
3211     }
3212     void clear();
3213     void report(String message);
3214     void reportError(String message);
3215     void reportError(ERROR_CODE errorCode, String message);
3216     void reportError(ERROR_CODE errorCode, String message, TableInfo table);
3217     void reportError(ERROR_CODE errorCode, String message, TableInfo table, HbckInfo info);
3218     void reportError(
3219       ERROR_CODE errorCode,
3220       String message,
3221       TableInfo table,
3222       HbckInfo info1,
3223       HbckInfo info2
3224     );
3225     int summarize();
3226     void detail(String details);
3227     ArrayList<ERROR_CODE> getErrorList();
3228     void progress();
3229     void print(String message);
3230     void resetErrors();
3231     boolean tableHasErrors(TableInfo table);
3232   }
3233 
3234   static class PrintingErrorReporter implements ErrorReporter {
3235     public int errorCount = 0;
3236     private int showProgress;
3237 
3238     Set<TableInfo> errorTables = new HashSet<TableInfo>();
3239 
3240     // for use by unit tests to verify which errors were discovered
3241     private ArrayList<ERROR_CODE> errorList = new ArrayList<ERROR_CODE>();
3242 
3243     @Override
3244     public void clear() {
3245       errorTables.clear();
3246       errorList.clear();
3247       errorCount = 0;
3248     }
3249 
3250     @Override
3251     public synchronized void reportError(ERROR_CODE errorCode, String message) {
3252       if (errorCode == ERROR_CODE.WRONG_USAGE) {
3253         System.err.println(message);
3254         return;
3255       }
3256 
3257       errorList.add(errorCode);
3258       if (!summary) {
3259         System.out.println("ERROR: " + message);
3260       }
3261       errorCount++;
3262       showProgress = 0;
3263     }
3264 
3265     @Override
3266     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
3267       errorTables.add(table);
3268       reportError(errorCode, message);
3269     }
3270 
3271     @Override
3272     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3273                                          HbckInfo info) {
3274       errorTables.add(table);
3275       String reference = "(region " + info.getRegionNameAsString() + ")";
3276       reportError(errorCode, reference + " " + message);
3277     }
3278 
3279     @Override
3280     public synchronized void reportError(ERROR_CODE errorCode, String message, TableInfo table,
3281                                          HbckInfo info1, HbckInfo info2) {
3282       errorTables.add(table);
3283       String reference = "(regions " + info1.getRegionNameAsString()
3284           + " and " + info2.getRegionNameAsString() + ")";
3285       reportError(errorCode, reference + " " + message);
3286     }
3287 
3288     @Override
3289     public synchronized void reportError(String message) {
3290       reportError(ERROR_CODE.UNKNOWN, message);
3291     }
3292 
3293     /**
3294      * Report error information, but do not increment the error count.  Intended for cases
3295      * where the actual error would have been reported previously.
3296      * @param message
3297      */
3298     @Override
3299     public synchronized void report(String message) {
3300       if (! summary) {
3301         System.out.println("ERROR: " + message);
3302       }
3303       showProgress = 0;
3304     }
3305 
3306     @Override
3307     public synchronized int summarize() {
3308       System.out.println(Integer.toString(errorCount) +
3309                          " inconsistencies detected.");
3310       if (errorCount == 0) {
3311         System.out.println("Status: OK");
3312         return 0;
3313       } else {
3314         System.out.println("Status: INCONSISTENT");
3315         return -1;
3316       }
3317     }
3318 
3319     @Override
3320     public ArrayList<ERROR_CODE> getErrorList() {
3321       return errorList;
3322     }
3323 
3324     @Override
3325     public synchronized void print(String message) {
3326       if (!summary) {
3327         System.out.println(message);
3328       }
3329     }
3330 
3331     @Override
3332     public boolean tableHasErrors(TableInfo table) {
3333       return errorTables.contains(table);
3334     }
3335 
3336     @Override
3337     public void resetErrors() {
3338       errorCount = 0;
3339     }
3340 
3341     @Override
3342     public synchronized void detail(String message) {
3343       if (details) {
3344         System.out.println(message);
3345       }
3346       showProgress = 0;
3347     }
3348 
3349     @Override
3350     public synchronized void progress() {
3351       if (showProgress++ == 10) {
3352         if (!summary) {
3353           System.out.print(".");
3354         }
3355         showProgress = 0;
3356       }
3357     }
3358   }
3359 
3360   /**
3361    * Contact a region server and get all information from it
3362    */
3363   static class WorkItemRegion implements Callable<Void> {
3364     private HBaseFsck hbck;
3365     private ServerName rsinfo;
3366     private ErrorReporter errors;
3367     private HConnection connection;
3368 
3369     WorkItemRegion(HBaseFsck hbck, ServerName info,
3370                    ErrorReporter errors, HConnection connection) {
3371       this.hbck = hbck;
3372       this.rsinfo = info;
3373       this.errors = errors;
3374       this.connection = connection;
3375     }
3376 
3377     @Override
3378     public synchronized Void call() throws IOException {
3379       errors.progress();
3380       try {
3381         BlockingInterface server = connection.getAdmin(rsinfo);
3382 
3383         // list all online regions from this region server
3384         List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
3385         regions = filterRegions(regions);
3386 
3387         if (details) {
3388           errors.detail("RegionServer: " + rsinfo.getServerName() +
3389                            " number of regions: " + regions.size());
3390           for (HRegionInfo rinfo: regions) {
3391             errors.detail("  " + rinfo.getRegionNameAsString() +
3392                              " id: " + rinfo.getRegionId() +
3393                              " encoded_name: " + rinfo.getEncodedName() +
3394                              " start: " + Bytes.toStringBinary(rinfo.getStartKey()) +
3395                              " end: " + Bytes.toStringBinary(rinfo.getEndKey()));
3396           }
3397         }
3398 
3399         // check to see if the existence of this region matches the region in META
3400         for (HRegionInfo r:regions) {
3401           HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());
3402           if (!RegionReplicaUtil.isDefaultReplica(r)) hbi.setSkipChecks(true);
3403           hbi.addServer(r, rsinfo);
3404         }
3405       } catch (IOException e) {          // unable to connect to the region server.
3406         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() +
3407           " Unable to fetch region information. " + e);
3408         throw e;
3409       }
3410       return null;
3411     }
3412 
3413     private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
3414       List<HRegionInfo> ret = Lists.newArrayList();
3415       for (HRegionInfo hri : regions) {
3416         if (hri.isMetaTable() || (!hbck.checkMetaOnly
3417             && hbck.isTableIncluded(hri.getTable()))) {
3418           ret.add(hri);
3419         }
3420       }
3421       return ret;
3422     }
3423   }
3424 
3425   /**
3426    * Contact hdfs and get all information about specified table directory into
3427    * regioninfo list.
3428    */
3429   static class WorkItemHdfsDir implements Callable<Void> {
3430     private HBaseFsck hbck;
3431     private FileStatus tableDir;
3432     private ErrorReporter errors;
3433     private FileSystem fs;
3434 
3435     WorkItemHdfsDir(HBaseFsck hbck, FileSystem fs, ErrorReporter errors,
3436                     FileStatus status) {
3437       this.hbck = hbck;
3438       this.fs = fs;
3439       this.tableDir = status;
3440       this.errors = errors;
3441     }
3442 
3443     @Override
3444     public synchronized Void call() throws IOException {
3445       try {
3446         // level 2: <HBASE_DIR>/<table>/*
3447         FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());
3448         for (FileStatus regionDir : regionDirs) {
3449           String encodedName = regionDir.getPath().getName();
3450           // ignore directories that aren't hexadecimal
3451           if (!encodedName.toLowerCase().matches("[0-9a-f]+")) {
3452             continue;
3453           }
3454 
3455           LOG.debug("Loading region info from hdfs:"+ regionDir.getPath());
3456           HbckInfo hbi = hbck.getOrCreateInfo(encodedName);
3457           HdfsEntry he = new HdfsEntry();
3458           synchronized (hbi) {
3459             if (hbi.getHdfsRegionDir() != null) {
3460               errors.print("Directory " + encodedName + " duplicate??" +
3461                            hbi.getHdfsRegionDir());
3462             }
3463 
3464             he.hdfsRegionDir = regionDir.getPath();
3465             he.hdfsRegionDirModTime = regionDir.getModificationTime();
3466             Path regioninfoFile = new Path(he.hdfsRegionDir, HRegionFileSystem.REGION_INFO_FILE);
3467             he.hdfsRegioninfoFilePresent = fs.exists(regioninfoFile);
3468             // we add to orphan list when we attempt to read .regioninfo
3469 
3470             // Set a flag if this region contains only edits
3471             // This is special case if a region is left after split
3472             he.hdfsOnlyEdits = true;
3473             FileStatus[] subDirs = fs.listStatus(regionDir.getPath());
3474             Path ePath = HLogUtil.getRegionDirRecoveredEditsDir(regionDir.getPath());
3475             for (FileStatus subDir : subDirs) {
3476               String sdName = subDir.getPath().getName();
3477               if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) {
3478                 he.hdfsOnlyEdits = false;
3479                 break;
3480               }
3481             }
3482             hbi.hdfsEntry = he;
3483           }
3484         }
3485       } catch (IOException e) {
3486         // unable to connect to the region server.
3487         errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "
3488             + tableDir.getPath().getName()
3489             + " Unable to fetch region information. " + e);
3490         throw e;
3491       }
3492       return null;
3493     }
3494   }
3495 
3496   /**
3497    * Contact hdfs and get all information about specified table directory into
3498    * regioninfo list.
3499    */
3500   static class WorkItemHdfsRegionInfo implements Callable<Void> {
3501     private HbckInfo hbi;
3502     private HBaseFsck hbck;
3503     private ErrorReporter errors;
3504 
3505     WorkItemHdfsRegionInfo(HbckInfo hbi, HBaseFsck hbck, ErrorReporter errors) {
3506       this.hbi = hbi;
3507       this.hbck = hbck;
3508       this.errors = errors;
3509     }
3510 
3511     @Override
3512     public synchronized Void call() throws IOException {
3513       // only load entries that haven't been loaded yet.
3514       if (hbi.getHdfsHRI() == null) {
3515         try {
3516           hbck.loadHdfsRegioninfo(hbi);
3517         } catch (IOException ioe) {
3518           String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "
3519               + hbi.getTableName() + " in hdfs dir "
3520               + hbi.getHdfsRegionDir()
3521               + "!  It may be an invalid format or version file.  Treating as "
3522               + "an orphaned regiondir.";
3523           errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);
3524           try {
3525             hbck.debugLsr(hbi.getHdfsRegionDir());
3526           } catch (IOException ioe2) {
3527             LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);
3528             throw ioe2;
3529           }
3530           hbck.orphanHdfsDirs.add(hbi);
3531           throw ioe;
3532         }
3533       }
3534       return null;
3535     }
3536   };
3537 
3538   /**
3539    * Display the full report from fsck. This displays all live and dead region
3540    * servers, and all known regions.
3541    */
3542   public static void setDisplayFullReport() {
3543     details = true;
3544   }
3545 
3546   /**
3547    * Set summary mode.
3548    * Print only summary of the tables and status (OK or INCONSISTENT)
3549    */
3550   void setSummary() {
3551     summary = true;
3552   }
3553 
3554   /**
3555    * Set hbase:meta check mode.
3556    * Print only info about hbase:meta table deployment/state
3557    */
3558   void setCheckMetaOnly() {
3559     checkMetaOnly = true;
3560   }
3561 
3562   /**
3563    * Set region boundaries check mode.
3564    */
3565   void setRegionBoundariesCheck() {
3566     checkRegionBoundaries = true;
3567   }
3568 
3569   /**
3570    * Set table locks fix mode.
3571    * Delete table locks held for a long time
3572    */
3573   public void setFixTableLocks(boolean shouldFix) {
3574     fixTableLocks = shouldFix;
3575   }
3576 
3577   /**
3578    * Check if we should rerun fsck again. This checks if we've tried to
3579    * fix something and we should rerun fsck tool again.
3580    * Display the full report from fsck. This displays all live and dead
3581    * region servers, and all known regions.
3582    */
3583   void setShouldRerun() {
3584     rerun = true;
3585   }
3586 
3587   boolean shouldRerun() {
3588     return rerun;
3589   }
3590 
3591   /**
3592    * Fix inconsistencies found by fsck. This should try to fix errors (if any)
3593    * found by fsck utility.
3594    */
3595   public void setFixAssignments(boolean shouldFix) {
3596     fixAssignments = shouldFix;
3597   }
3598 
3599   boolean shouldFixAssignments() {
3600     return fixAssignments;
3601   }
3602 
3603   public void setFixMeta(boolean shouldFix) {
3604     fixMeta = shouldFix;
3605   }
3606 
3607   boolean shouldFixMeta() {
3608     return fixMeta;
3609   }
3610 
3611   public void setFixEmptyMetaCells(boolean shouldFix) {
3612     fixEmptyMetaCells = shouldFix;
3613   }
3614 
3615   boolean shouldFixEmptyMetaCells() {
3616     return fixEmptyMetaCells;
3617   }
3618 
3619   public void setCheckHdfs(boolean checking) {
3620     checkHdfs = checking;
3621   }
3622 
3623   boolean shouldCheckHdfs() {
3624     return checkHdfs;
3625   }
3626 
3627   public void setFixHdfsHoles(boolean shouldFix) {
3628     fixHdfsHoles = shouldFix;
3629   }
3630 
3631   boolean shouldFixHdfsHoles() {
3632     return fixHdfsHoles;
3633   }
3634 
3635   public void setFixTableOrphans(boolean shouldFix) {
3636     fixTableOrphans = shouldFix;
3637   }
3638 
3639   boolean shouldFixTableOrphans() {
3640     return fixTableOrphans;
3641   }
3642 
3643   public void setFixHdfsOverlaps(boolean shouldFix) {
3644     fixHdfsOverlaps = shouldFix;
3645   }
3646 
3647   boolean shouldFixHdfsOverlaps() {
3648     return fixHdfsOverlaps;
3649   }
3650 
3651   public void setFixHdfsOrphans(boolean shouldFix) {
3652     fixHdfsOrphans = shouldFix;
3653   }
3654 
3655   boolean shouldFixHdfsOrphans() {
3656     return fixHdfsOrphans;
3657   }
3658 
3659   public void setFixVersionFile(boolean shouldFix) {
3660     fixVersionFile = shouldFix;
3661   }
3662 
3663   public boolean shouldFixVersionFile() {
3664     return fixVersionFile;
3665   }
3666 
3667   public void setSidelineBigOverlaps(boolean sbo) {
3668     this.sidelineBigOverlaps = sbo;
3669   }
3670 
3671   public boolean shouldSidelineBigOverlaps() {
3672     return sidelineBigOverlaps;
3673   }
3674 
3675   public void setFixSplitParents(boolean shouldFix) {
3676     fixSplitParents = shouldFix;
3677   }
3678 
3679   boolean shouldFixSplitParents() {
3680     return fixSplitParents;
3681   }
3682 
3683   public void setFixReferenceFiles(boolean shouldFix) {
3684     fixReferenceFiles = shouldFix;
3685   }
3686 
3687   boolean shouldFixReferenceFiles() {
3688     return fixReferenceFiles;
3689   }
3690 
3691   public boolean shouldIgnorePreCheckPermission() {
3692     return ignorePreCheckPermission;
3693   }
3694 
3695   public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {
3696     this.ignorePreCheckPermission = ignorePreCheckPermission;
3697   }
3698 
3699   /**
3700    * @param mm maximum number of regions to merge into a single region.
3701    */
3702   public void setMaxMerge(int mm) {
3703     this.maxMerge = mm;
3704   }
3705 
3706   public int getMaxMerge() {
3707     return maxMerge;
3708   }
3709 
3710   public void setMaxOverlapsToSideline(int mo) {
3711     this.maxOverlapsToSideline = mo;
3712   }
3713 
3714   public int getMaxOverlapsToSideline() {
3715     return maxOverlapsToSideline;
3716   }
3717 
3718   /**
3719    * Only check/fix tables specified by the list,
3720    * Empty list means all tables are included.
3721    */
3722   boolean isTableIncluded(TableName table) {
3723     return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
3724   }
3725 
3726   public void includeTable(TableName table) {
3727     tablesIncluded.add(table);
3728   }
3729 
3730   Set<TableName> getIncludedTables() {
3731     return new HashSet<TableName>(tablesIncluded);
3732   }
3733 
3734   /**
3735    * We are interested in only those tables that have not changed their state in
3736    * hbase:meta during the last few seconds specified by hbase.admin.fsck.timelag
3737    * @param seconds - the time in seconds
3738    */
3739   public void setTimeLag(long seconds) {
3740     timelag = seconds * 1000; // convert to milliseconds
3741   }
3742 
3743   /**
3744    *
3745    * @param sidelineDir - HDFS path to sideline data
3746    */
3747   public void setSidelineDir(String sidelineDir) {
3748     this.sidelineDir = new Path(sidelineDir);
3749   }
3750 
3751   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
3752     return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
3753   }
3754 
3755   public HFileCorruptionChecker getHFilecorruptionChecker() {
3756     return hfcc;
3757   }
3758 
3759   public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {
3760     this.hfcc = hfcc;
3761   }
3762 
3763   public void setRetCode(int code) {
3764     this.retcode = code;
3765   }
3766 
3767   public int getRetCode() {
3768     return retcode;
3769   }
3770 
3771   protected HBaseFsck printUsageAndExit() {
3772     StringWriter sw = new StringWriter(2048);
3773     PrintWriter out = new PrintWriter(sw);
3774     out.println("Usage: fsck [opts] {only tables}");
3775     out.println(" where [opts] are:");
3776     out.println("   -help Display help options (this)");
3777     out.println("   -details Display full report of all regions.");
3778     out.println("   -timelag <timeInSeconds>  Process only regions that " +
3779                        " have not experienced any metadata updates in the last " +
3780                        " <timeInSeconds> seconds.");
3781     out.println("   -sleepBeforeRerun <timeInSeconds> Sleep this many seconds" +
3782         " before checking if the fix worked if run with -fix");
3783     out.println("   -summary Print only summary of the tables and status.");
3784     out.println("   -metaonly Only check the state of the hbase:meta table.");
3785     out.println("   -sidelineDir <hdfs://> HDFS path to backup existing meta.");
3786     out.println("   -boundaries Verify that regions boundaries are the same between META and store files.");
3787 
3788     out.println("");
3789     out.println("  Metadata Repair options: (expert features, use with caution!)");
3790     out.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
3791     out.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
3792     out.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
3793     out.println("   -noHdfsChecking   Don't load/check region info from HDFS."
3794         + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
3795     out.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
3796     out.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
3797     out.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
3798     out.println("   -fixHdfsOverlaps  Try to fix region overlaps in hdfs.");
3799     out.println("   -fixVersionFile   Try to fix missing hbase.version file in hdfs.");
3800     out.println("   -maxMerge <n>     When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
3801     out.println("   -sidelineBigOverlaps  When fixing region overlaps, allow to sideline big overlaps");
3802     out.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
3803     out.println("   -fixSplitParents  Try to force offline split parents to be online.");
3804     out.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
3805     out.println("   -fixReferenceFiles  Try to offline lingering reference store files");
3806     out.println("   -fixEmptyMetaCells  Try to fix hbase:meta entries not referencing any region"
3807         + " (empty REGIONINFO_QUALIFIER rows)");
3808 
3809     out.println("");
3810     out.println("  Datafile Repair options: (expert features, use with caution!)");
3811     out.println("   -checkCorruptHFiles     Check all Hfiles by opening them to make sure they are valid");
3812     out.println("   -sidelineCorruptHFiles  Quarantine corrupted HFiles.  implies -checkCorruptHFiles");
3813 
3814     out.println("");
3815     out.println("  Metadata Repair shortcuts");
3816     out.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
3817         "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles -fixTableLocks");
3818     out.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
3819 
3820     out.println("");
3821     out.println("  Table lock options");
3822     out.println("   -fixTableLocks    Deletes table locks held for a long time (hbase.table.lock.expire.ms, 10min by default)");
3823 
3824     out.flush();
3825     errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());
3826 
3827     setRetCode(-2);
3828     return this;
3829   }
3830 
3831   /**
3832    * Main program
3833    *
3834    * @param args
3835    * @throws Exception
3836    */
3837   public static void main(String[] args) throws Exception {
3838     // create a fsck object
3839     Configuration conf = HBaseConfiguration.create();
3840     Path hbasedir = FSUtils.getRootDir(conf);
3841     URI defaultFs = hbasedir.getFileSystem(conf).getUri();
3842     FSUtils.setFsDefault(conf, new Path(defaultFs));
3843 
3844     int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
3845     System.exit(ret);
3846   }
3847 
3848   /**
3849    * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.
3850    */
3851   static class HBaseFsckTool extends Configured implements Tool {
3852     HBaseFsckTool(Configuration conf) { super(conf); }
3853     @Override
3854     public int run(String[] args) throws Exception {
3855       HBaseFsck hbck = new HBaseFsck(getConf());
3856       hbck.exec(hbck.executor, args);
3857       return hbck.getRetCode();
3858     }
3859   };
3860 
3861 
3862   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
3863     ServiceException, InterruptedException {
3864     long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
3865 
3866     boolean checkCorruptHFiles = false;
3867     boolean sidelineCorruptHFiles = false;
3868 
3869     // Process command-line args.
3870     for (int i = 0; i < args.length; i++) {
3871       String cmd = args[i];
3872       if (cmd.equals("-help") || cmd.equals("-h")) {
3873         return printUsageAndExit();
3874       } else if (cmd.equals("-details")) {
3875         setDisplayFullReport();
3876       } else if (cmd.equals("-timelag")) {
3877         if (i == args.length - 1) {
3878           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");
3879           return printUsageAndExit();
3880         }
3881         try {
3882           long timelag = Long.parseLong(args[i+1]);
3883           setTimeLag(timelag);
3884         } catch (NumberFormatException e) {
3885           errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");
3886           return printUsageAndExit();
3887         }
3888         i++;
3889       } else if (cmd.equals("-sleepBeforeRerun")) {
3890         if (i == args.length - 1) {
3891           errors.reportError(ERROR_CODE.WRONG_USAGE,
3892             "HBaseFsck: -sleepBeforeRerun needs a value.");
3893           return printUsageAndExit();
3894         }
3895         try {
3896           sleepBeforeRerun = Long.parseLong(args[i+1]);
3897         } catch (NumberFormatException e) {
3898           errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");
3899           return printUsageAndExit();
3900         }
3901         i++;
3902       } else if (cmd.equals("-sidelineDir")) {
3903         if (i == args.length - 1) {
3904           errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");
3905           return printUsageAndExit();
3906         }
3907         i++;
3908         setSidelineDir(args[i]);
3909       } else if (cmd.equals("-fix")) {
3910         errors.reportError(ERROR_CODE.WRONG_USAGE,
3911           "This option is deprecated, please use  -fixAssignments instead.");
3912         setFixAssignments(true);
3913       } else if (cmd.equals("-fixAssignments")) {
3914         setFixAssignments(true);
3915       } else if (cmd.equals("-fixMeta")) {
3916         setFixMeta(true);
3917       } else if (cmd.equals("-noHdfsChecking")) {
3918         setCheckHdfs(false);
3919       } else if (cmd.equals("-fixHdfsHoles")) {
3920         setFixHdfsHoles(true);
3921       } else if (cmd.equals("-fixHdfsOrphans")) {
3922         setFixHdfsOrphans(true);
3923       } else if (cmd.equals("-fixTableOrphans")) {
3924         setFixTableOrphans(true);
3925       } else if (cmd.equals("-fixHdfsOverlaps")) {
3926         setFixHdfsOverlaps(true);
3927       } else if (cmd.equals("-fixVersionFile")) {
3928         setFixVersionFile(true);
3929       } else if (cmd.equals("-sidelineBigOverlaps")) {
3930         setSidelineBigOverlaps(true);
3931       } else if (cmd.equals("-fixSplitParents")) {
3932         setFixSplitParents(true);
3933       } else if (cmd.equals("-ignorePreCheckPermission")) {
3934         setIgnorePreCheckPermission(true);
3935       } else if (cmd.equals("-checkCorruptHFiles")) {
3936         checkCorruptHFiles = true;
3937       } else if (cmd.equals("-sidelineCorruptHFiles")) {
3938         sidelineCorruptHFiles = true;
3939       } else if (cmd.equals("-fixReferenceFiles")) {
3940         setFixReferenceFiles(true);
3941       } else if (cmd.equals("-fixEmptyMetaCells")) {
3942         setFixEmptyMetaCells(true);
3943       } else if (cmd.equals("-repair")) {
3944         // this attempts to merge overlapping hdfs regions, needs testing
3945         // under load
3946         setFixHdfsHoles(true);
3947         setFixHdfsOrphans(true);
3948         setFixMeta(true);
3949         setFixAssignments(true);
3950         setFixHdfsOverlaps(true);
3951         setFixVersionFile(true);
3952         setSidelineBigOverlaps(true);
3953         setFixSplitParents(false);
3954         setCheckHdfs(true);
3955         setFixReferenceFiles(true);
3956         setFixTableLocks(true);
3957       } else if (cmd.equals("-repairHoles")) {
3958         // this will make all missing hdfs regions available but may lose data
3959         setFixHdfsHoles(true);
3960         setFixHdfsOrphans(false);
3961         setFixMeta(true);
3962         setFixAssignments(true);
3963         setFixHdfsOverlaps(false);
3964         setSidelineBigOverlaps(false);
3965         setFixSplitParents(false);
3966         setCheckHdfs(true);
3967       } else if (cmd.equals("-maxOverlapsToSideline")) {
3968         if (i == args.length - 1) {
3969           errors.reportError(ERROR_CODE.WRONG_USAGE,
3970             "-maxOverlapsToSideline needs a numeric value argument.");
3971           return printUsageAndExit();
3972         }
3973         try {
3974           int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
3975           setMaxOverlapsToSideline(maxOverlapsToSideline);
3976         } catch (NumberFormatException e) {
3977           errors.reportError(ERROR_CODE.WRONG_USAGE,
3978             "-maxOverlapsToSideline needs a numeric value argument.");
3979           return printUsageAndExit();
3980         }
3981         i++;
3982       } else if (cmd.equals("-maxMerge")) {
3983         if (i == args.length - 1) {
3984           errors.reportError(ERROR_CODE.WRONG_USAGE,
3985             "-maxMerge needs a numeric value argument.");
3986           return printUsageAndExit();
3987         }
3988         try {
3989           int maxMerge = Integer.parseInt(args[i+1]);
3990           setMaxMerge(maxMerge);
3991         } catch (NumberFormatException e) {
3992           errors.reportError(ERROR_CODE.WRONG_USAGE,
3993             "-maxMerge needs a numeric value argument.");
3994           return printUsageAndExit();
3995         }
3996         i++;
3997       } else if (cmd.equals("-summary")) {
3998         setSummary();
3999       } else if (cmd.equals("-metaonly")) {
4000         setCheckMetaOnly();
4001       } else if (cmd.equals("-boundaries")) {
4002         setRegionBoundariesCheck();
4003       } else if (cmd.equals("-fixTableLocks")) {
4004         setFixTableLocks(true);
4005       } else if (cmd.startsWith("-")) {
4006         errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);
4007         return printUsageAndExit();
4008       } else {
4009         includeTable(TableName.valueOf(cmd));
4010         errors.print("Allow checking/fixes for table: " + cmd);
4011       }
4012     }
4013 
4014     // pre-check current user has FS write permission or not
4015     try {
4016       preCheckPermission();
4017     } catch (AccessControlException ace) {
4018       Runtime.getRuntime().exit(-1);
4019     } catch (IOException ioe) {
4020       Runtime.getRuntime().exit(-1);
4021     }
4022 
4023     // do the real work of hbck
4024     connect();
4025 
4026     try {
4027       // if corrupt file mode is on, first fix them since they may be opened later
4028       if (checkCorruptHFiles || sidelineCorruptHFiles) {
4029         LOG.info("Checking all hfiles for corruption");
4030         HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
4031         setHFileCorruptionChecker(hfcc); // so we can get result
4032         Collection<TableName> tables = getIncludedTables();
4033         Collection<Path> tableDirs = new ArrayList<Path>();
4034         Path rootdir = FSUtils.getRootDir(getConf());
4035         if (tables.size() > 0) {
4036           for (TableName t : tables) {
4037             tableDirs.add(FSUtils.getTableDir(rootdir, t));
4038           }
4039         } else {
4040           tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
4041         }
4042         hfcc.checkTables(tableDirs);
4043         hfcc.report(errors);
4044       }
4045 
4046       // check and fix table integrity, region consistency.
4047       int code = onlineHbck();
4048       setRetCode(code);
4049       // If we have changed the HBase state it is better to run hbck again
4050       // to see if we haven't broken something else in the process.
4051       // We run it only once more because otherwise we can easily fall into
4052       // an infinite loop.
4053       if (shouldRerun()) {
4054         try {
4055           LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");
4056           Thread.sleep(sleepBeforeRerun);
4057         } catch (InterruptedException ie) {
4058           LOG.warn("Interrupted while sleeping");
4059           return this;
4060         }
4061         // Just report
4062         setFixAssignments(false);
4063         setFixMeta(false);
4064         setFixHdfsHoles(false);
4065         setFixHdfsOverlaps(false);
4066         setFixVersionFile(false);
4067         setFixTableOrphans(false);
4068         errors.resetErrors();
4069         code = onlineHbck();
4070         setRetCode(code);
4071       }
4072     } finally {
4073       IOUtils.cleanup(null, connection, meta, admin);
4074     }
4075     return this;
4076   }
4077 
4078   /**
4079    * ls -r for debugging purposes
4080    */
4081   void debugLsr(Path p) throws IOException {
4082     debugLsr(getConf(), p, errors);
4083   }
4084 
4085   /**
4086    * ls -r for debugging purposes
4087    */
4088   public static void debugLsr(Configuration conf,
4089       Path p) throws IOException {
4090     debugLsr(conf, p, new PrintingErrorReporter());
4091   }
4092 
4093   /**
4094    * ls -r for debugging purposes
4095    */
4096   public static void debugLsr(Configuration conf,
4097       Path p, ErrorReporter errors) throws IOException {
4098     if (!LOG.isDebugEnabled() || p == null) {
4099       return;
4100     }
4101     FileSystem fs = p.getFileSystem(conf);
4102 
4103     if (!fs.exists(p)) {
4104       // nothing
4105       return;
4106     }
4107     errors.print(p.toString());
4108 
4109     if (fs.isFile(p)) {
4110       return;
4111     }
4112 
4113     if (fs.getFileStatus(p).isDirectory()) {
4114       FileStatus[] fss= fs.listStatus(p);
4115       for (FileStatus status : fss) {
4116         debugLsr(conf, status.getPath(), errors);
4117       }
4118     }
4119   }
4120 }